486 lines
24 KiB
C#
486 lines
24 KiB
C#
using System;
|
|
using System.Collections.Concurrent;
|
|
using System.Collections.Generic;
|
|
using System.IO;
|
|
using System.IO.Compression;
|
|
using System.Linq;
|
|
using System.Threading;
|
|
using System.Threading.Tasks;
|
|
using HtmlAgilityPack;
|
|
using MediatR;
|
|
using Microsoft.AspNetCore.Authorization;
|
|
using Microsoft.AspNetCore.Http;
|
|
using Microsoft.AspNetCore.Mvc;
|
|
using Microsoft.AspNetCore.SignalR;
|
|
using Knot.Modules.Chats.Application.Abstractions;
|
|
using Knot.Modules.Chats.Domain;
|
|
using Knot.Modules.Identity.Domain;
|
|
using Knot.Shared.Kernel;
|
|
using Knot.Shared.Kernel.Storage;
|
|
using Knot.Modules.Chats.Infrastructure.SignalR;
|
|
using Knot.Modules.Chats.Application.Chats.Create;
|
|
|
|
namespace Host.Controllers;
|
|
|
|
[Authorize]
|
|
[ApiController]
|
|
[Route("api/import/telegram")]
|
|
public sealed class TelegramImportController : ControllerBase
|
|
{
|
|
private readonly ISender _sender;
|
|
private readonly IUserContext _userContext;
|
|
private readonly IChatsUnitOfWork _unitOfWork;
|
|
private readonly IChatRepository _chatRepository;
|
|
private readonly IMessageRepository _messageRepository;
|
|
private readonly IFileStorageService _fileStorage;
|
|
private readonly IHubContext<ChatHub> _hubContext;
|
|
|
|
// In-memory store for uploaded zips (good enough for typical cases, ideally should be removed after use or by timer)
|
|
private static readonly ConcurrentDictionary<Guid, string> _tempZips = new();
|
|
|
|
public TelegramImportController(
|
|
ISender sender,
|
|
IUserContext userContext,
|
|
IChatsUnitOfWork unitOfWork,
|
|
IChatRepository chatRepository,
|
|
IMessageRepository messageRepository,
|
|
IFileStorageService fileStorage,
|
|
IHubContext<ChatHub> hubContext)
|
|
{
|
|
_sender = sender;
|
|
_userContext = userContext;
|
|
_unitOfWork = unitOfWork;
|
|
_chatRepository = chatRepository;
|
|
_messageRepository = messageRepository;
|
|
_fileStorage = fileStorage;
|
|
_hubContext = hubContext;
|
|
}
|
|
|
|
[HttpPost("analyze")]
|
|
[DisableRequestSizeLimit]
|
|
[RequestFormLimits(MultipartBodyLengthLimit = 10L * 1024 * 1024 * 1024)] // 10GB for big exports
|
|
public async Task<IActionResult> Analyze(IFormFile file, CancellationToken ct)
|
|
{
|
|
if (file == null || file.Length == 0) return BadRequest("No file uploaded");
|
|
if (!file.FileName.EndsWith(".zip", StringComparison.OrdinalIgnoreCase)) return BadRequest("Must be a ZIP archive");
|
|
|
|
var token = Guid.NewGuid();
|
|
var tempPath = Path.Combine(Path.GetTempPath(), $"{token}.zip");
|
|
|
|
await using (var fs = new FileStream(tempPath, FileMode.Create))
|
|
{
|
|
await file.CopyToAsync(fs, ct);
|
|
}
|
|
|
|
var names = new HashSet<string>();
|
|
|
|
// Open zip and quickly scan messages.html
|
|
using (var archive = ZipFile.OpenRead(tempPath))
|
|
{
|
|
var htmlEntries = archive.Entries
|
|
.Where(e => e.FullName.EndsWith(".html", StringComparison.OrdinalIgnoreCase) && e.Name.StartsWith("messages", StringComparison.OrdinalIgnoreCase));
|
|
|
|
foreach (var entry in htmlEntries)
|
|
{
|
|
using var stream = entry.Open();
|
|
var parser = new AngleSharp.Html.Parser.HtmlParser();
|
|
var doc = parser.ParseDocument(stream);
|
|
|
|
var messageNodes = doc.QuerySelectorAll(".message");
|
|
if (messageNodes == null) continue;
|
|
|
|
foreach (var node in messageNodes)
|
|
{
|
|
var fromNameNode = node.QuerySelector(".from_name");
|
|
if (fromNameNode != null)
|
|
{
|
|
var nameNodeText = (AngleSharp.Dom.IElement)fromNameNode.Clone();
|
|
var innerSpans = nameNodeText.QuerySelectorAll("span");
|
|
foreach (var span in innerSpans) span.Remove();
|
|
|
|
var name = nameNodeText.TextContent.Trim();
|
|
if (!string.IsNullOrWhiteSpace(name))
|
|
{
|
|
names.Add(name);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
_tempZips[token] = tempPath;
|
|
|
|
return Ok(new
|
|
{
|
|
token,
|
|
names = names.ToList()
|
|
});
|
|
}
|
|
|
|
public sealed record ExecuteImportRequest(Guid Token, Dictionary<string, Guid> Mapping, string? GroupName);
|
|
|
|
[HttpPost("execute")]
|
|
public async Task<IActionResult> Execute([FromBody] ExecuteImportRequest req, CancellationToken ct)
|
|
{
|
|
if (!_tempZips.TryGetValue(req.Token, out var tempPath))
|
|
return BadRequest("Session not found or expired");
|
|
|
|
if (!System.IO.File.Exists(tempPath))
|
|
return BadRequest("ZIP file lost");
|
|
|
|
var myId = _userContext.UserId;
|
|
// Collect targeted users to check whose chat it is. Find the friend.
|
|
// Usually, the mapping contains MyId and FriendId.
|
|
var targetUserIds = req.Mapping.Values.Distinct().Where(id => id != Guid.Empty).ToList();
|
|
if (!targetUserIds.Contains(myId)) targetUserIds.Add(myId);
|
|
|
|
Guid chatId = Guid.Empty;
|
|
var chatMembers = targetUserIds;
|
|
|
|
if (chatMembers.Count <= 2)
|
|
{
|
|
// Find existing personal chat
|
|
var existingChats = await _chatRepository.GetUserChatsAsync(myId, ct);
|
|
var personalChat = existingChats.FirstOrDefault(c => c.Type == ChatType.Personal && c.Members.All(m => chatMembers.Contains(m.UserId)) && c.Members.Count == chatMembers.Count);
|
|
|
|
if (personalChat != null)
|
|
{
|
|
chatId = personalChat.Id;
|
|
}
|
|
else
|
|
{
|
|
// Create personal chat
|
|
var friendId = chatMembers.FirstOrDefault(id => id != myId);
|
|
if (friendId == Guid.Empty) friendId = myId; // Notes to self
|
|
var command = new CreateChatCommand(string.Empty, ChatType.Personal, new List<Guid> { myId, friendId });
|
|
var res = await _sender.Send(command, ct);
|
|
if (res.IsFailure) return BadRequest(res.Error);
|
|
chatId = res.Value;
|
|
}
|
|
}
|
|
else
|
|
{
|
|
// Create a group
|
|
var command = new CreateChatCommand(req.GroupName ?? "Импортированный чат", ChatType.Group, chatMembers);
|
|
var res = await _sender.Send(command, ct);
|
|
if (res.IsFailure) return BadRequest(res.Error);
|
|
chatId = res.Value;
|
|
}
|
|
|
|
int importedCount = 0;
|
|
|
|
using (var archive = ZipFile.OpenRead(tempPath))
|
|
{
|
|
var htmlEntries = archive.Entries
|
|
.Where(e => e.FullName.EndsWith(".html", StringComparison.OrdinalIgnoreCase) && e.Name.StartsWith("messages", StringComparison.OrdinalIgnoreCase))
|
|
.OrderBy(e =>
|
|
{
|
|
var name = e.Name.ToLower().Replace("messages", "").Replace(".html", "");
|
|
return string.IsNullOrEmpty(name) ? 0 : int.TryParse(name, out var num) ? num : 999999;
|
|
})
|
|
.ToList();
|
|
|
|
Guid lastSenderGuid = myId;
|
|
DateTime lastCreatedAt = DateTime.UtcNow;
|
|
Dictionary<string, Guid> messageIdMap = new();
|
|
Message? lastSavedMessage = null;
|
|
|
|
foreach (var entry in htmlEntries)
|
|
{
|
|
using var stream = entry.Open();
|
|
var parser = new AngleSharp.Html.Parser.HtmlParser();
|
|
var doc = parser.ParseDocument(stream);
|
|
|
|
var messageNodes = doc.QuerySelectorAll(".message");
|
|
if (messageNodes == null) continue;
|
|
|
|
var baseDir = Path.GetDirectoryName(entry.FullName)?.Replace("\\", "/") ?? "";
|
|
if (!string.IsNullOrEmpty(baseDir) && !baseDir.EndsWith("/")) baseDir += "/";
|
|
|
|
foreach (var node in messageNodes)
|
|
{
|
|
try
|
|
{
|
|
var fromNameNode = node.QuerySelector(".from_name");
|
|
var textNode = node.QuerySelector(".text");
|
|
|
|
var dateNode = node.QuerySelector(".date[title]")
|
|
?? node.QuerySelector(".pull_right[title]")
|
|
?? node.QuerySelector("[title]");
|
|
|
|
if (fromNameNode != null)
|
|
{
|
|
var nameNodeText = (AngleSharp.Dom.IElement)fromNameNode.Clone();
|
|
var innerSpans = nameNodeText.QuerySelectorAll("span");
|
|
foreach (var span in innerSpans) span.Remove();
|
|
|
|
var name = nameNodeText.TextContent.Trim();
|
|
if (req.Mapping.TryGetValue(name, out var mappedId) && mappedId != Guid.Empty)
|
|
lastSenderGuid = mappedId;
|
|
else
|
|
lastSenderGuid = myId;
|
|
}
|
|
|
|
Guid senderGuid = lastSenderGuid;
|
|
|
|
string content = "";
|
|
var mainBodyNode = node.QuerySelector(".body");
|
|
var isForwarded = node.QuerySelector(".forwarded") != null;
|
|
|
|
// To avoid grabbing text from inside the forwarded block as main text,
|
|
// we can look for .text that is a direct child of the main .body
|
|
// The .forwarded block has its own .text
|
|
var contentTextNode = isForwarded
|
|
? (node.QuerySelector(".body > .text") ?? node.QuerySelector(".text:not(.forwarded .text)"))
|
|
: node.QuerySelector(".text");
|
|
|
|
if (contentTextNode != null)
|
|
{
|
|
var html = contentTextNode.InnerHtml
|
|
.Replace("<br>", "\n", StringComparison.OrdinalIgnoreCase)
|
|
.Replace("<br/>", "\n", StringComparison.OrdinalIgnoreCase)
|
|
.Replace("<br />", "\n", StringComparison.OrdinalIgnoreCase);
|
|
var tempParser = new AngleSharp.Html.Parser.HtmlParser();
|
|
var tempDoc = tempParser.ParseDocument("<div>" + html + "</div>");
|
|
content = tempDoc.Body?.TextContent.Trim() ?? "";
|
|
}
|
|
|
|
DateTime createdAt = lastCreatedAt;
|
|
var titleNodes = node.QuerySelectorAll("[title]");
|
|
bool parsed = false;
|
|
|
|
if (titleNodes != null)
|
|
{
|
|
foreach (var tnode in titleNodes)
|
|
{
|
|
var dateStr = tnode.GetAttribute("title")?.Trim() ?? "";
|
|
|
|
if (dateStr.Length >= 10 && char.IsDigit(dateStr[0]) && char.IsDigit(dateStr[1]))
|
|
{
|
|
// Remove "UTC" so we can parse timezone offset generically, e.g. "17.10.2025 12:56:25 +03:00"
|
|
var cleanStr = dateStr.Replace("UTC", "", StringComparison.OrdinalIgnoreCase).Trim();
|
|
|
|
if (DateTimeOffset.TryParseExact(cleanStr, "dd.MM.yyyy HH:mm:ss zzz", System.Globalization.CultureInfo.InvariantCulture, System.Globalization.DateTimeStyles.None, out var dto))
|
|
{
|
|
createdAt = dto.UtcDateTime;
|
|
parsed = true;
|
|
break;
|
|
}
|
|
else if (DateTime.TryParseExact(cleanStr, "dd.MM.yyyy HH:mm:ss", System.Globalization.CultureInfo.InvariantCulture, System.Globalization.DateTimeStyles.AssumeUniversal | System.Globalization.DateTimeStyles.AdjustToUniversal, out var dt))
|
|
{
|
|
createdAt = dt;
|
|
parsed = true;
|
|
break;
|
|
}
|
|
else if (DateTime.TryParse(cleanStr, out var dFallback))
|
|
{
|
|
createdAt = dFallback.ToUniversalTime();
|
|
parsed = true;
|
|
break;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
if (!parsed)
|
|
{
|
|
Console.WriteLine("Warning: Could not parse date in imported message! Using lastCreatedAt.");
|
|
}
|
|
else
|
|
{
|
|
lastCreatedAt = createdAt;
|
|
}
|
|
|
|
// Parse forwards
|
|
Guid? forwardedFromId = null;
|
|
var forwardedNode = node.QuerySelector(".forwarded.body");
|
|
if (forwardedNode != null)
|
|
{
|
|
var fwdNameNode = forwardedNode.QuerySelector(".from_name");
|
|
var fwdNameText = fwdNameNode != null ? (AngleSharp.Dom.IElement)fwdNameNode.Clone() : null;
|
|
if (fwdNameText != null)
|
|
{
|
|
var innerSpans = fwdNameText.QuerySelectorAll("span");
|
|
foreach (var s in innerSpans) s.Remove();
|
|
}
|
|
var fwdName = fwdNameText != null ? fwdNameText.TextContent.Trim() : "Неизвестного";
|
|
|
|
if (req.Mapping.TryGetValue(fwdName, out var mappedFwdId) && mappedFwdId != Guid.Empty)
|
|
{
|
|
forwardedFromId = mappedFwdId;
|
|
}
|
|
else if (fwdName == "Это я" || fwdName == req.Mapping.FirstOrDefault(x => x.Value == myId).Key)
|
|
{
|
|
forwardedFromId = myId;
|
|
}
|
|
|
|
var fwdTextNode = forwardedNode.QuerySelector(".text");
|
|
string fwdContent = "";
|
|
if (fwdTextNode != null)
|
|
{
|
|
var fHtml = fwdTextNode.InnerHtml
|
|
.Replace("<br>", "\n", StringComparison.OrdinalIgnoreCase)
|
|
.Replace("<br/>", "\n", StringComparison.OrdinalIgnoreCase)
|
|
.Replace("<br />", "\n", StringComparison.OrdinalIgnoreCase);
|
|
var tempParser = new AngleSharp.Html.Parser.HtmlParser();
|
|
var tempDoc = tempParser.ParseDocument("<div>" + fHtml + "</div>");
|
|
fwdContent = tempDoc.Body?.TextContent.Trim() ?? "";
|
|
}
|
|
|
|
if (forwardedFromId == null)
|
|
{
|
|
// We don't have this user in the app, map to generic string
|
|
content = string.IsNullOrEmpty(content)
|
|
? $"[Переслано от {fwdName}]:\n{fwdContent}"
|
|
: $"{content}\n\n[Переслано от {fwdName}]:\n{fwdContent}";
|
|
}
|
|
else if (string.IsNullOrEmpty(content))
|
|
{
|
|
content = fwdContent;
|
|
}
|
|
}
|
|
|
|
// Parse replies
|
|
Guid? replyToId = null;
|
|
var replyNode = node.QuerySelector(".reply_to a");
|
|
if (replyNode != null)
|
|
{
|
|
var href = replyNode.GetAttribute("href");
|
|
if (href != null && href.StartsWith("#go_to_"))
|
|
{
|
|
var tgId = href.Substring("#go_to_".Length);
|
|
if (messageIdMap.TryGetValue(tgId, out var mappedMsgId))
|
|
{
|
|
replyToId = mappedMsgId;
|
|
}
|
|
}
|
|
}
|
|
|
|
var mediaNodes = node.QuerySelectorAll("a.photo_wrap, a.animated_wrap, video, audio, a.document, a.media_voice_message, a.media_video, img.sticker").ToList();
|
|
if (mediaNodes.Count == 0)
|
|
{
|
|
var fallback = node.QuerySelectorAll(".media_wrap a[href]");
|
|
mediaNodes.AddRange(fallback);
|
|
}
|
|
|
|
var messageType = "text";
|
|
|
|
// Helper for categorizing files
|
|
(string mType, string cType) GetMediaTypes(string fileUrl)
|
|
{
|
|
var ext = Path.GetExtension(fileUrl)?.ToLower();
|
|
return ext switch
|
|
{
|
|
".jpg" or ".jpeg" or ".png" or ".webp" => ("image", "image/jpeg"),
|
|
".mp4" or ".mov" or ".avi" => ("video", "video/mp4"),
|
|
".ogg" or ".mp3" => ("voice", "audio/ogg"),
|
|
_ => ("file", "application/octet-stream")
|
|
};
|
|
}
|
|
|
|
if (mediaNodes != null && mediaNodes.Count > 0)
|
|
{
|
|
var firstHref = mediaNodes[0].GetAttribute("href") ?? mediaNodes[0].GetAttribute("src");
|
|
if (firstHref != null)
|
|
{
|
|
messageType = GetMediaTypes(firstHref).mType;
|
|
if (mediaNodes[0].ClassName?.Contains("animated") == true || firstHref.EndsWith(".mp4"))
|
|
{
|
|
// Treat telegram animated gifs as image format in app (we auto-loop mp4 images)
|
|
// But web app specifically handles "image" and "video" and microlink crashes on mp4 video
|
|
// Let's just keep "video" or "image". Actually, telegram exports gifs as mp4.
|
|
// We should let our app player handle it.
|
|
if (mediaNodes[0].ClassName?.Contains("animated") == true)
|
|
messageType = "image";
|
|
}
|
|
}
|
|
}
|
|
|
|
bool isJoined = fromNameNode == null;
|
|
bool isMediaOnly = string.IsNullOrEmpty(content) && forwardedNode == null && replyToId == null;
|
|
Message? targetMessage = null;
|
|
|
|
if (isJoined && isMediaOnly && lastSavedMessage != null && Math.Abs((createdAt - lastSavedMessage.CreatedAt).TotalSeconds) <= 60 && lastSavedMessage.SenderId == senderGuid)
|
|
{
|
|
targetMessage = lastSavedMessage;
|
|
}
|
|
else
|
|
{
|
|
string finalContent = content;
|
|
targetMessage = Message.Import(chatId, senderGuid, finalContent, messageType, createdAt, replyToId, forwardedFromId);
|
|
|
|
var idAttr = node.GetAttribute("id");
|
|
if (!string.IsNullOrEmpty(idAttr))
|
|
{
|
|
messageIdMap[idAttr] = targetMessage.Id;
|
|
}
|
|
}
|
|
|
|
if (mediaNodes != null)
|
|
{
|
|
foreach (var mediaNode in mediaNodes)
|
|
{
|
|
string? href = mediaNode.GetAttribute("href") ?? mediaNode.GetAttribute("src");
|
|
if (!string.IsNullOrEmpty(href) && !href.StartsWith("http"))
|
|
{
|
|
var zipPath = baseDir + href.Replace("\\", "/");
|
|
var zipEntry = archive.GetEntry(zipPath);
|
|
if (zipEntry != null)
|
|
{
|
|
using var ms = new MemoryStream();
|
|
using var zipfs = zipEntry.Open();
|
|
await zipfs.CopyToAsync(ms, ct);
|
|
ms.Position = 0;
|
|
|
|
var types = GetMediaTypes(href);
|
|
var finalMType = types.mType;
|
|
if (mediaNode.ClassName?.Contains("animated") == true) finalMType = "image";
|
|
var fileId = await _fileStorage.UploadFileAsync(ms, Path.GetFileName(href), types.cType);
|
|
targetMessage.AddMedia(finalMType, $"/api/files/{fileId}", Path.GetFileName(href), zipEntry.Length);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// Parse reactions
|
|
var reactionNodes = node.QuerySelectorAll(".reactions .reaction");
|
|
foreach (var reactionNode in reactionNodes)
|
|
{
|
|
var emojiNode = reactionNode.QuerySelector(".emoji");
|
|
if (emojiNode == null) continue;
|
|
var emoji = emojiNode.TextContent.Trim();
|
|
|
|
var userpicNodes = reactionNode.QuerySelectorAll(".userpics .userpic .initials[title]");
|
|
foreach (var userpicNode in userpicNodes)
|
|
{
|
|
var title = userpicNode.GetAttribute("title")?.Trim();
|
|
if (!string.IsNullOrEmpty(title) && req.Mapping.TryGetValue(title, out var rUserId) && rUserId != Guid.Empty)
|
|
{
|
|
targetMessage.AddReaction(rUserId, emoji);
|
|
}
|
|
}
|
|
}
|
|
if (targetMessage != lastSavedMessage && (!string.IsNullOrEmpty(content) || targetMessage.Media.Any()))
|
|
{
|
|
_messageRepository.Add(targetMessage);
|
|
lastSavedMessage = targetMessage;
|
|
importedCount++;
|
|
}
|
|
}
|
|
catch { /* ignore single message parse error */ }
|
|
}
|
|
}
|
|
}
|
|
|
|
await _unitOfWork.SaveChangesAsync(ct);
|
|
|
|
// Delete temp zip
|
|
try { System.IO.File.Delete(tempPath); _tempZips.TryRemove(req.Token, out _); } catch { }
|
|
|
|
// Notify UI for all members of the chat
|
|
await _hubContext.Clients.Users(chatMembers.Select(x => x.ToString())).SendAsync("history_updated", new { chatId });
|
|
|
|
return Ok(new { success = true, messagesImported = importedCount, chatId });
|
|
}
|
|
}
|