using System; using System.Collections.Concurrent; using System.Collections.Generic; using System.IO; using System.IO.Compression; using System.Linq; using System.Threading; using System.Threading.Tasks; using HtmlAgilityPack; using MediatR; using Microsoft.AspNetCore.Authorization; using Microsoft.AspNetCore.Http; using Microsoft.AspNetCore.Mvc; using Microsoft.AspNetCore.SignalR; using Knot.Modules.Chats.Application.Abstractions; using Knot.Modules.Chats.Domain; using Knot.Modules.Identity.Domain; using Knot.Shared.Kernel; using Knot.Shared.Kernel.Storage; using Knot.Modules.Chats.Infrastructure.SignalR; using Knot.Modules.Chats.Application.Chats.Create; namespace Host.Controllers; [Authorize] [ApiController] [Route("api/import/telegram")] public sealed class TelegramImportController : ControllerBase { private readonly ISender _sender; private readonly IUserContext _userContext; private readonly IChatsUnitOfWork _unitOfWork; private readonly IChatRepository _chatRepository; private readonly IMessageRepository _messageRepository; private readonly IFileStorageService _fileStorage; private readonly IHubContext _hubContext; // In-memory store for uploaded zips (good enough for typical cases, ideally should be removed after use or by timer) private static readonly ConcurrentDictionary _tempZips = new(); public TelegramImportController( ISender sender, IUserContext userContext, IChatsUnitOfWork unitOfWork, IChatRepository chatRepository, IMessageRepository messageRepository, IFileStorageService fileStorage, IHubContext hubContext) { _sender = sender; _userContext = userContext; _unitOfWork = unitOfWork; _chatRepository = chatRepository; _messageRepository = messageRepository; _fileStorage = fileStorage; _hubContext = hubContext; } [HttpPost("analyze")] [DisableRequestSizeLimit] [RequestFormLimits(MultipartBodyLengthLimit = 10L * 1024 * 1024 * 1024)] // 10GB for big exports public async Task Analyze(IFormFile file, CancellationToken ct) { if (file == null || file.Length == 0) return BadRequest("No file uploaded"); if (!file.FileName.EndsWith(".zip", StringComparison.OrdinalIgnoreCase)) return BadRequest("Must be a ZIP archive"); var token = Guid.NewGuid(); var tempPath = Path.Combine(Path.GetTempPath(), $"{token}.zip"); await using (var fs = new FileStream(tempPath, FileMode.Create)) { await file.CopyToAsync(fs, ct); } var names = new HashSet(); // Open zip and quickly scan messages.html using (var archive = ZipFile.OpenRead(tempPath)) { var htmlEntries = archive.Entries .Where(e => e.FullName.EndsWith(".html", StringComparison.OrdinalIgnoreCase) && e.Name.StartsWith("messages", StringComparison.OrdinalIgnoreCase)); foreach (var entry in htmlEntries) { using var stream = entry.Open(); var parser = new AngleSharp.Html.Parser.HtmlParser(); var doc = parser.ParseDocument(stream); var messageNodes = doc.QuerySelectorAll(".message"); if (messageNodes == null) continue; foreach (var node in messageNodes) { var fromNameNode = node.QuerySelector(".from_name"); if (fromNameNode != null) { var nameNodeText = (AngleSharp.Dom.IElement)fromNameNode.Clone(); var innerSpans = nameNodeText.QuerySelectorAll("span"); foreach (var span in innerSpans) span.Remove(); var name = nameNodeText.TextContent.Trim(); if (!string.IsNullOrWhiteSpace(name)) { names.Add(name); } } } } } _tempZips[token] = tempPath; return Ok(new { token, names = names.ToList() }); } public sealed record ExecuteImportRequest(Guid Token, Dictionary Mapping, string? GroupName); [HttpPost("execute")] public async Task Execute([FromBody] ExecuteImportRequest req, CancellationToken ct) { if (!_tempZips.TryGetValue(req.Token, out var tempPath)) return BadRequest("Session not found or expired"); if (!System.IO.File.Exists(tempPath)) return BadRequest("ZIP file lost"); var myId = _userContext.UserId; // Collect targeted users to check whose chat it is. Find the friend. // Usually, the mapping contains MyId and FriendId. var targetUserIds = req.Mapping.Values.Distinct().Where(id => id != Guid.Empty).ToList(); if (!targetUserIds.Contains(myId)) targetUserIds.Add(myId); Guid chatId = Guid.Empty; var chatMembers = targetUserIds; if (chatMembers.Count <= 2) { // Find existing personal chat var existingChats = await _chatRepository.GetUserChatsAsync(myId, ct); var personalChat = existingChats.FirstOrDefault(c => c.Type == ChatType.Personal && c.Members.All(m => chatMembers.Contains(m.UserId)) && c.Members.Count == chatMembers.Count); if (personalChat != null) { chatId = personalChat.Id; } else { // Create personal chat var friendId = chatMembers.FirstOrDefault(id => id != myId); if (friendId == Guid.Empty) friendId = myId; // Notes to self var command = new CreateChatCommand(string.Empty, ChatType.Personal, new List { myId, friendId }); var res = await _sender.Send(command, ct); if (res.IsFailure) return BadRequest(res.Error); chatId = res.Value; } } else { // Create a group var command = new CreateChatCommand(req.GroupName ?? "Импортированный чат", ChatType.Group, chatMembers); var res = await _sender.Send(command, ct); if (res.IsFailure) return BadRequest(res.Error); chatId = res.Value; } int importedCount = 0; using (var archive = ZipFile.OpenRead(tempPath)) { var htmlEntries = archive.Entries .Where(e => e.FullName.EndsWith(".html", StringComparison.OrdinalIgnoreCase) && e.Name.StartsWith("messages", StringComparison.OrdinalIgnoreCase)) .OrderBy(e => { var name = e.Name.ToLower().Replace("messages", "").Replace(".html", ""); return string.IsNullOrEmpty(name) ? 0 : int.TryParse(name, out var num) ? num : 999999; }) .ToList(); Guid lastSenderGuid = myId; DateTime lastCreatedAt = DateTime.UtcNow; Dictionary messageIdMap = new(); Message? lastSavedMessage = null; foreach (var entry in htmlEntries) { using var stream = entry.Open(); var parser = new AngleSharp.Html.Parser.HtmlParser(); var doc = parser.ParseDocument(stream); var messageNodes = doc.QuerySelectorAll(".message"); if (messageNodes == null) continue; var baseDir = Path.GetDirectoryName(entry.FullName)?.Replace("\\", "/") ?? ""; if (!string.IsNullOrEmpty(baseDir) && !baseDir.EndsWith("/")) baseDir += "/"; foreach (var node in messageNodes) { try { var fromNameNode = node.QuerySelector(".from_name"); var textNode = node.QuerySelector(".text"); var dateNode = node.QuerySelector(".date[title]") ?? node.QuerySelector(".pull_right[title]") ?? node.QuerySelector("[title]"); if (fromNameNode != null) { var nameNodeText = (AngleSharp.Dom.IElement)fromNameNode.Clone(); var innerSpans = nameNodeText.QuerySelectorAll("span"); foreach (var span in innerSpans) span.Remove(); var name = nameNodeText.TextContent.Trim(); if (req.Mapping.TryGetValue(name, out var mappedId) && mappedId != Guid.Empty) lastSenderGuid = mappedId; else lastSenderGuid = myId; } Guid senderGuid = lastSenderGuid; string content = ""; var mainBodyNode = node.QuerySelector(".body"); var isForwarded = node.QuerySelector(".forwarded") != null; // To avoid grabbing text from inside the forwarded block as main text, // we can look for .text that is a direct child of the main .body // The .forwarded block has its own .text var contentTextNode = isForwarded ? (node.QuerySelector(".body > .text") ?? node.QuerySelector(".text:not(.forwarded .text)")) : node.QuerySelector(".text"); if (contentTextNode != null) { var html = contentTextNode.InnerHtml .Replace("
", "\n", StringComparison.OrdinalIgnoreCase) .Replace("
", "\n", StringComparison.OrdinalIgnoreCase) .Replace("
", "\n", StringComparison.OrdinalIgnoreCase); var tempParser = new AngleSharp.Html.Parser.HtmlParser(); var tempDoc = tempParser.ParseDocument("
" + html + "
"); content = tempDoc.Body?.TextContent.Trim() ?? ""; } DateTime createdAt = lastCreatedAt; var titleNodes = node.QuerySelectorAll("[title]"); bool parsed = false; if (titleNodes != null) { foreach (var tnode in titleNodes) { var dateStr = tnode.GetAttribute("title")?.Trim() ?? ""; if (dateStr.Length >= 10 && char.IsDigit(dateStr[0]) && char.IsDigit(dateStr[1])) { // Remove "UTC" so we can parse timezone offset generically, e.g. "17.10.2025 12:56:25 +03:00" var cleanStr = dateStr.Replace("UTC", "", StringComparison.OrdinalIgnoreCase).Trim(); if (DateTimeOffset.TryParseExact(cleanStr, "dd.MM.yyyy HH:mm:ss zzz", System.Globalization.CultureInfo.InvariantCulture, System.Globalization.DateTimeStyles.None, out var dto)) { createdAt = dto.UtcDateTime; parsed = true; break; } else if (DateTime.TryParseExact(cleanStr, "dd.MM.yyyy HH:mm:ss", System.Globalization.CultureInfo.InvariantCulture, System.Globalization.DateTimeStyles.AssumeUniversal | System.Globalization.DateTimeStyles.AdjustToUniversal, out var dt)) { createdAt = dt; parsed = true; break; } else if (DateTime.TryParse(cleanStr, out var dFallback)) { createdAt = dFallback.ToUniversalTime(); parsed = true; break; } } } } if (!parsed) { Console.WriteLine("Warning: Could not parse date in imported message! Using lastCreatedAt."); } else { lastCreatedAt = createdAt; } // Parse forwards Guid? forwardedFromId = null; var forwardedNode = node.QuerySelector(".forwarded.body"); if (forwardedNode != null) { var fwdNameNode = forwardedNode.QuerySelector(".from_name"); var fwdNameText = fwdNameNode != null ? (AngleSharp.Dom.IElement)fwdNameNode.Clone() : null; if (fwdNameText != null) { var innerSpans = fwdNameText.QuerySelectorAll("span"); foreach (var s in innerSpans) s.Remove(); } var fwdName = fwdNameText != null ? fwdNameText.TextContent.Trim() : "Неизвестного"; if (req.Mapping.TryGetValue(fwdName, out var mappedFwdId) && mappedFwdId != Guid.Empty) { forwardedFromId = mappedFwdId; } else if (fwdName == "Это я" || fwdName == req.Mapping.FirstOrDefault(x => x.Value == myId).Key) { forwardedFromId = myId; } var fwdTextNode = forwardedNode.QuerySelector(".text"); string fwdContent = ""; if (fwdTextNode != null) { var fHtml = fwdTextNode.InnerHtml .Replace("
", "\n", StringComparison.OrdinalIgnoreCase) .Replace("
", "\n", StringComparison.OrdinalIgnoreCase) .Replace("
", "\n", StringComparison.OrdinalIgnoreCase); var tempParser = new AngleSharp.Html.Parser.HtmlParser(); var tempDoc = tempParser.ParseDocument("
" + fHtml + "
"); fwdContent = tempDoc.Body?.TextContent.Trim() ?? ""; } if (forwardedFromId == null) { // We don't have this user in the app, map to generic string content = string.IsNullOrEmpty(content) ? $"[Переслано от {fwdName}]:\n{fwdContent}" : $"{content}\n\n[Переслано от {fwdName}]:\n{fwdContent}"; } else if (string.IsNullOrEmpty(content)) { content = fwdContent; } } // Parse replies Guid? replyToId = null; var replyNode = node.QuerySelector(".reply_to a"); if (replyNode != null) { var href = replyNode.GetAttribute("href"); if (href != null && href.StartsWith("#go_to_")) { var tgId = href.Substring("#go_to_".Length); if (messageIdMap.TryGetValue(tgId, out var mappedMsgId)) { replyToId = mappedMsgId; } } } var mediaNodes = node.QuerySelectorAll("a.photo_wrap, a.animated_wrap, video, audio, a.document, a.media_voice_message, a.media_video, img.sticker").ToList(); if (mediaNodes.Count == 0) { var fallback = node.QuerySelectorAll(".media_wrap a[href]"); mediaNodes.AddRange(fallback); } var messageType = "text"; // Helper for categorizing files (string mType, string cType) GetMediaTypes(string fileUrl) { var ext = Path.GetExtension(fileUrl)?.ToLower(); return ext switch { ".jpg" or ".jpeg" or ".png" or ".webp" => ("image", "image/jpeg"), ".mp4" or ".mov" or ".avi" => ("video", "video/mp4"), ".ogg" or ".mp3" => ("voice", "audio/ogg"), _ => ("file", "application/octet-stream") }; } if (mediaNodes != null && mediaNodes.Count > 0) { var firstHref = mediaNodes[0].GetAttribute("href") ?? mediaNodes[0].GetAttribute("src"); if (firstHref != null) { messageType = GetMediaTypes(firstHref).mType; if (mediaNodes[0].ClassName?.Contains("animated") == true || firstHref.EndsWith(".mp4")) { // Treat telegram animated gifs as image format in app (we auto-loop mp4 images) // But web app specifically handles "image" and "video" and microlink crashes on mp4 video // Let's just keep "video" or "image". Actually, telegram exports gifs as mp4. // We should let our app player handle it. if (mediaNodes[0].ClassName?.Contains("animated") == true) messageType = "image"; } } } bool isJoined = fromNameNode == null; bool isMediaOnly = string.IsNullOrEmpty(content) && forwardedNode == null && replyToId == null; Message? targetMessage = null; if (isJoined && isMediaOnly && lastSavedMessage != null && Math.Abs((createdAt - lastSavedMessage.CreatedAt).TotalSeconds) <= 60 && lastSavedMessage.SenderId == senderGuid) { targetMessage = lastSavedMessage; } else { string finalContent = content; targetMessage = Message.Import(chatId, senderGuid, finalContent, messageType, createdAt, replyToId, forwardedFromId); var idAttr = node.GetAttribute("id"); if (!string.IsNullOrEmpty(idAttr)) { messageIdMap[idAttr] = targetMessage.Id; } } if (mediaNodes != null) { foreach (var mediaNode in mediaNodes) { string? href = mediaNode.GetAttribute("href") ?? mediaNode.GetAttribute("src"); if (!string.IsNullOrEmpty(href) && !href.StartsWith("http")) { var zipPath = baseDir + href.Replace("\\", "/"); var zipEntry = archive.GetEntry(zipPath); if (zipEntry != null) { using var ms = new MemoryStream(); using var zipfs = zipEntry.Open(); await zipfs.CopyToAsync(ms, ct); ms.Position = 0; var types = GetMediaTypes(href); var finalMType = types.mType; if (mediaNode.ClassName?.Contains("animated") == true) finalMType = "image"; var fileId = await _fileStorage.UploadFileAsync(ms, Path.GetFileName(href), types.cType); targetMessage.AddMedia(finalMType, $"/api/files/{fileId}", Path.GetFileName(href), zipEntry.Length); } } } } // Parse reactions var reactionNodes = node.QuerySelectorAll(".reactions .reaction"); foreach (var reactionNode in reactionNodes) { var emojiNode = reactionNode.QuerySelector(".emoji"); if (emojiNode == null) continue; var emoji = emojiNode.TextContent.Trim(); var userpicNodes = reactionNode.QuerySelectorAll(".userpics .userpic .initials[title]"); foreach (var userpicNode in userpicNodes) { var title = userpicNode.GetAttribute("title")?.Trim(); if (!string.IsNullOrEmpty(title) && req.Mapping.TryGetValue(title, out var rUserId) && rUserId != Guid.Empty) { targetMessage.AddReaction(rUserId, emoji); } } } if (targetMessage != lastSavedMessage && (!string.IsNullOrEmpty(content) || targetMessage.Media.Any())) { _messageRepository.Add(targetMessage); lastSavedMessage = targetMessage; importedCount++; } } catch { /* ignore single message parse error */ } } } } await _unitOfWork.SaveChangesAsync(ct); // Delete temp zip try { System.IO.File.Delete(tempPath); _tempZips.TryRemove(req.Token, out _); } catch { } // Notify UI for all members of the chat await _hubContext.Clients.Users(chatMembers.Select(x => x.ToString())).SendAsync("history_updated", new { chatId }); return Ok(new { success = true, messagesImported = importedCount, chatId }); } }