Further admin api work

2026-05-15 05:15:17 +00:00 · 2025-04-10 17:12:53 +02:00
parent abb1b570a4
commit e1483e9f90
17 changed files with 642 additions and 20 deletions
@@ -24,6 +24,11 @@
                <span class="bi bi-list-nested-nav-menu" aria-hidden="true"></span> Guilds
            </NavLink>
        </div>
+        <div class="nav-item px-3">
+            <NavLink class="nav-link" href="Media">
+                <span class="bi bi-list-nested-nav-menu" aria-hidden="true"></span> Media
+            </NavLink>
+        </div>
    </nav>
 </div>

@@ -0,0 +1,7 @@
+@page "/Media"
+<h3>Index of /Media</h3>
+<hr/>
+
+@code {
+    
+}
@@ -0,0 +1,106 @@
+@page "/Media/ByUser"
+@using System.Net.Http.Headers
+@using System.Reflection
+@using Spacebar.AdminApi.Models
+@using Spacebar.AdminAPI.TestClient.Services
+@using ArcaneLibs.Blazor.Components
+@inject Config Config
+@inject ILocalStorageService LocalStorage
+
+<PageTitle>Uploaded media by user</PageTitle>
+
+<details>
+    <summary>Displayed columns</summary>
+    @foreach (var column in DisplayedColumns) {
+        var value = column.Value;
+        <span>
+            <InputCheckbox @bind-Value:get="@(value)" @bind-Value:set="@(b => {
+                                                                           DisplayedColumns[column.Key] = b;
+                                                                           StateHasChanged();
+                                                                       })"/>
+            @column.Key.Name
+        </span>
+        <br/>
+    }
+</details>
+
+<InputSelect @bind-Value="@SelectedUserId">
+    <option value="">All users</option>
+    @if (UserList is { Count: > 0 }) {
+        @foreach (var user in UserList.OrderByDescending(u => u.Id).Where(x => !x.Deleted)) {
+            <option value="@user.Id">@user.Username</option>
+        }
+    }
+</InputSelect>
+
+
+<table class="table table-bordered">
+    @{
+        var columns = DisplayedColumns.Where(kvp => kvp.Value).Select(kvp => kvp.Key).ToList();
+    }
+    <thead>
+        <tr>
+            @foreach (var column in columns) {
+                <th>@column.Name</th>
+            }
+            <th>Actions</th>
+        </tr>
+    </thead>
+    <tbody>
+        @foreach (var user in UserMedia) {
+            <tr>
+                @foreach (var column in columns) {
+                    <td>@column.GetValue(user)</td>
+                }
+                <td>
+                    <LinkButton href="@($"/Users/Delete/{user.Id}")" Color="#ff0000">Delete</LinkButton>
+                </td>
+            </tr>
+        }
+    </tbody>
+</table>
+
+@code {
+
+    private Dictionary<PropertyInfo, bool> DisplayedColumns { get; set; } = typeof(FileMetadataModel).GetProperties()
+        .ToDictionary(p => p, p => p.Name == "Username" || p.Name == "Id" || p.Name == "MessageCount");
+
+    private List<UserModel> UserList { get; set; } = new();
+    private List<FileMetadataModel> UserMedia { get; set; } = new();
+
+    [SupplyParameterFromQuery(Name = "UserId")]
+    public string? SelectedUserId {
+        get;
+        set {
+            field = value;
+            if (string.IsNullOrWhiteSpace(field))
+                UserMedia.Clear();
+            else _ = GetMediaForUser(value!);
+        }
+    }
+
+    protected override async Task OnInitializedAsync() {
+        using var hc = new HttpClient();
+        hc.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", Config.AccessToken);
+        var response = await hc.GetAsync(Config.AdminUrl + "/_spacebar/admin/users/");
+        if (!response.IsSuccessStatusCode) throw new Exception(await response.Content.ReadAsStringAsync());
+        var content = response.Content.ReadFromJsonAsAsyncEnumerable<UserModel>();
+        await foreach (var user in content) {
+            UserList.Add(user!);
+            StateHasChanged();
+        }
+    }
+
+    private async Task GetMediaForUser(string userId) {
+        using var hc = new HttpClient();
+        hc.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", Config.AccessToken);
+        var response = await hc.GetAsync(Config.AdminUrl + $"/_spacebar/admin/media/user/{userId}/attachments");
+        if (!response.IsSuccessStatusCode) throw new Exception(await response.Content.ReadAsStringAsync());
+        var content = response.Content.ReadFromJsonAsAsyncEnumerable<FileMetadataModel>();
+        await foreach (var media in content) {
+            UserMedia.Add(media!);
+            StateHasChanged();
+        }
+    }
+
+}
@@ -4,6 +4,7 @@
@using Spacebar.AdminApi.Models
@using Spacebar.AdminAPI.TestClient.Services
@using ArcaneLibs.Blazor.Components
+@using ArcaneLibs.Extensions
@inject Config Config
@inject ILocalStorageService LocalStorage

@@ -24,6 +25,7 @@
    }
 </details>

+<p>Got @UserList.Count users.</p>
 <table class="table table-bordered">
    @{
        var columns = DisplayedColumns.Where(kvp => kvp.Value).Select(kvp => kvp.Key).ToList();
@@ -37,7 +39,7 @@
        </tr>
    </thead>
    <tbody>
-        @foreach (var user in UserList) {
+        @foreach (var user in UserList.Where(x => !x.Deleted).OrderByDescending(x=>x.MessageCount)) {
            <tr>
                @foreach (var column in columns) {
                    <td>@column.GetValue(user)</td>
@@ -58,15 +60,21 @@
    private List<UserModel> UserList { get; set; } = new();

    protected override async Task OnInitializedAsync() {
-        using var hc = new HttpClient();
+        var hc = new StreamingHttpClient();
        hc.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", Config.AccessToken);
-        var response = await hc.GetAsync(Config.AdminUrl + "/_spacebar/admin/users/");
-        if (!response.IsSuccessStatusCode) throw new Exception(await response.Content.ReadAsStringAsync());
-        var content = response.Content.ReadFromJsonAsAsyncEnumerable<UserModel>();
-        await foreach (var user in content) {
-            UserList.Add(user);
-            StateHasChanged();
+
+        // var request = new HttpRequestMessage(HttpMethod.Get, Config.AdminUrl + "/_spacebar/admin/users/");
+        
+        var response = hc.GetAsyncEnumerableFromJsonAsync<UserModel>(Config.AdminUrl + "/_spacebar/admin/users/");
+        // if (!response.IsSuccessStatusCode) throw new Exception(await response.Content.ReadAsStringAsync());
+        // var content = response.Content.ReadFromJsonAsAsyncEnumerable<UserModel>();
+        await foreach (var user in response) {
+            // Console.WriteLine(user.ToJson(indent: false, ignoreNull: true));
+            UserList.Add(user!);
+            if(UserList.Count % 1000 == 0)
+                StateHasChanged();
        }
+        StateHasChanged();
    }

 }
@@ -14,15 +14,21 @@ Deleted @ChannelDeleteProgress.Sum(x=>x.Value.Deleted) messages so far!
    <progress max="@progress.Total" value="@progress.Deleted"></progress>
 }

+@if (Done) {
+    <h1>Done!</h1>
+}
+
@code {

    [Parameter]
    public required string Id { get; set; }

    private Dictionary<string, DeleteProgress> ChannelDeleteProgress { get; set; } = new();
+    
+    private bool Done { get; set; }

    protected override async Task OnInitializedAsync() {
-        using var hc = new HttpClient();
+        var hc = new StreamingHttpClient();
        hc.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", Config.AccessToken);
        var response = await hc.GetAsync(Config.AdminUrl + $"/_spacebar/admin/Users/{Id}/delete?messageDeleteChunkSize=100");
        if (!response.IsSuccessStatusCode) throw new Exception(await response.Content.ReadAsStringAsync());
@@ -47,10 +53,13 @@ Deleted @ChannelDeleteProgress.Sum(x=>x.Value.Deleted) messages so far!
                    break;
                }
            }
-
+            
            StateHasChanged();
            await Task.Delay(1);
        }
+
+        Done = true;
+        StateHasChanged();
    }

    private class DeleteProgress {
@@ -0,0 +1,296 @@
+#define SINGLE_HTTPCLIENT // Use a single HttpClient instance for all MatrixHttpClient instances
+// #define SYNC_HTTPCLIENT   // Only allow one request as a time, for debugging
+using System.Data;
+using System.Diagnostics;
+using System.Diagnostics.CodeAnalysis;
+using System.Net.Http.Headers;
+using System.Reflection;
+using System.Text;
+using System.Text.Json;
+using System.Text.Json.Serialization;
+using ArcaneLibs;
+using ArcaneLibs.Extensions;
+
+namespace Spacebar.AdminAPI.TestClient.Services;
+
+#if SINGLE_HTTPCLIENT
+// TODO: Add URI wrapper for
+public class StreamingHttpClient {
+    private static readonly HttpClient Client;
+
+    static StreamingHttpClient() {
+        try {
+            var handler = new SocketsHttpHandler {
+                PooledConnectionLifetime = TimeSpan.FromMinutes(15),
+                MaxConnectionsPerServer = 4096,
+                EnableMultipleHttp2Connections = true
+            };
+            Client = new HttpClient(handler) {
+                DefaultRequestVersion = new Version(3, 0),
+                Timeout = TimeSpan.FromDays(1)
+            };
+        }
+        catch (PlatformNotSupportedException e) {
+            Console.WriteLine("Failed to create HttpClient with connection pooling, continuing without connection pool!");
+            Console.WriteLine("Original exception (safe to ignore!):");
+            Console.WriteLine(e);
+
+            Client = new HttpClient {
+                DefaultRequestVersion = new Version(3, 0)
+            };
+        }
+        catch (Exception e) {
+            Console.WriteLine("Failed to create HttpClient:");
+            Console.WriteLine(e);
+            throw;
+        }
+    }
+
+#if SYNC_HTTPCLIENT
+    internal SemaphoreSlim _rateLimitSemaphore { get; } = new(1, 1);
+#endif
+
+    public static bool LogRequests = true;
+    public Dictionary<string, string> AdditionalQueryParameters { get; set; } = new();
+
+    public Uri? BaseAddress { get; set; }
+
+    // default headers, not bound to client
+    public HttpRequestHeaders DefaultRequestHeaders { get; set; } =
+        typeof(HttpRequestHeaders).GetConstructor(BindingFlags.NonPublic | BindingFlags.Instance, null, [], null)?.Invoke([]) as HttpRequestHeaders ??
+        throw new InvalidOperationException("Failed to create HttpRequestHeaders");
+
+    private static JsonSerializerOptions GetJsonSerializerOptions(JsonSerializerOptions? options = null) {
+        options ??= new JsonSerializerOptions();
+        // options.Converters.Add(new JsonFloatStringConverter());
+        // options.Converters.Add(new JsonDoubleStringConverter());
+        // options.Converters.Add(new JsonDecimalStringConverter());
+        options.DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull;
+        return options;
+    }
+
+    public async Task<HttpResponseMessage> SendUnhandledAsync(HttpRequestMessage request, CancellationToken cancellationToken) {
+        if (request.RequestUri is null) throw new NullReferenceException("RequestUri is null");
+        // if (!request.RequestUri.IsAbsoluteUri)
+        request.RequestUri = request.RequestUri.EnsureAbsolute(BaseAddress!);
+        var swWait = Stopwatch.StartNew();
+#if SYNC_HTTPCLIENT
+        await _rateLimitSemaphore.WaitAsync(cancellationToken);
+#endif
+
+        if (request.RequestUri is null) throw new NullReferenceException("RequestUri is null");
+        if (!request.RequestUri.IsAbsoluteUri)
+            request.RequestUri = new Uri(BaseAddress ?? throw new InvalidOperationException("Relative URI passed, but no BaseAddress is specified!"), request.RequestUri);
+        swWait.Stop();
+        var swExec = Stopwatch.StartNew();
+        
+        foreach (var (key, value) in AdditionalQueryParameters) request.RequestUri = request.RequestUri.AddQuery(key, value);
+        foreach (var (key, value) in DefaultRequestHeaders) {
+            if (request.Headers.Contains(key)) continue;
+            request.Headers.Add(key, value);
+        }
+
+        request.Options.Set(new HttpRequestOptionsKey<bool>("WebAssemblyEnableStreamingResponse"), true);
+
+        if (LogRequests)
+            Console.WriteLine("Sending " + request.Summarise(includeHeaders: true, includeQuery: true, includeContentIfText: false, hideHeaders: ["Accept"]));
+
+        HttpResponseMessage? responseMessage;
+        try {
+            responseMessage = await Client.SendAsync(request, HttpCompletionOption.ResponseHeadersRead, cancellationToken);
+        }
+        catch (Exception e) {
+            if (e is TaskCanceledException or TimeoutException) {
+                if (request.Method == HttpMethod.Get && !cancellationToken.IsCancellationRequested) {
+                    await Task.Delay(Random.Shared.Next(500, 2500), cancellationToken);
+                    request.ResetSendStatus();
+                    return await SendAsync(request, cancellationToken);
+                }
+            }
+            else if (!e.ToString().StartsWith("TypeError: NetworkError"))
+                Console.WriteLine(
+                    $"Failed to send request {request.Method} {BaseAddress}{request.RequestUri} ({Util.BytesToString(request.Content?.Headers.ContentLength ?? 0)}):\n{e}");
+            throw;
+        }
+#if SYNC_HTTPCLIENT
+        finally {
+            _rateLimitSemaphore.Release();
+        }
+#endif
+
+        // Console.WriteLine($"Sending {request.Method} {request.RequestUri} ({Util.BytesToString(request.Content?.Headers.ContentLength ?? 0)}) -> {(int)responseMessage.StatusCode} {responseMessage.StatusCode} ({Util.BytesToString(responseMessage.GetContentLength())}, WAIT={swWait.ElapsedMilliseconds}ms, EXEC={swExec.ElapsedMilliseconds}ms)");
+        if (LogRequests)
+            Console.WriteLine("Received " + responseMessage.Summarise(includeHeaders: true, includeContentIfText: false, hideHeaders: [
+                "Server",
+                "Date",
+                "Transfer-Encoding",
+                "Connection",
+                "Vary",
+                "Content-Length",
+                "Access-Control-Allow-Origin",
+                "Access-Control-Allow-Methods",
+                "Access-Control-Allow-Headers",
+                "Access-Control-Expose-Headers",
+                "Cache-Control",
+                "Cross-Origin-Resource-Policy",
+                "X-Content-Security-Policy",
+                "Referrer-Policy",
+                "X-Robots-Tag",
+                "Content-Security-Policy"
+            ]));
+
+        return responseMessage;
+    }
+
+    public async Task<HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken cancellationToken = default) {
+        var responseMessage = await SendUnhandledAsync(request, cancellationToken);
+        if (responseMessage.IsSuccessStatusCode) return responseMessage;
+
+        //retry on gateway timeout
+        // if (responseMessage.StatusCode == HttpStatusCode.GatewayTimeout) {
+            // request.ResetSendStatus();
+            // return await SendAsync(request, cancellationToken);
+        // }
+
+        //error handling
+        var content = await responseMessage.Content.ReadAsStringAsync(cancellationToken);
+        if (content.Length == 0)
+            throw new DataException("Content was empty");
+            // throw new MatrixException() {
+                // ErrorCode = "M_UNKNOWN",
+                // Error = "Unknown error, server returned no content"
+            // };
+        
+        // if (!content.StartsWith('{')) throw new InvalidDataException("Encountered invalid data:\n" + content);
+        if (!content.TrimStart().StartsWith('{')) {
+            responseMessage.EnsureSuccessStatusCode();
+            throw new InvalidDataException("Encountered invalid data:\n" + content);
+        }
+        //we have a matrix error
+
+        throw new Exception("Unknown http exception");
+        // MatrixException? ex;
+        // try {
+        //     ex = JsonSerializer.Deserialize<MatrixException>(content);
+        // }
+        // catch (JsonException e) {
+        //     throw new LibMatrixException() {
+        //         ErrorCode = "M_INVALID_JSON",
+        //         Error = e.Message + "\nBody:\n" + await responseMessage.Content.ReadAsStringAsync(cancellationToken)
+        //     };
+        // }
+        //
+        // Debug.Assert(ex != null, nameof(ex) + " != null");
+        // ex.RawContent = content;
+        // // Console.WriteLine($"Failed to send request: {ex}");
+        // if (ex.RetryAfterMs is null) throw ex!;
+        // //we have a ratelimit error
+        // await Task.Delay(ex.RetryAfterMs.Value, cancellationToken);
+        request.ResetSendStatus();
+        return await SendAsync(request, cancellationToken);
+    }
+
+    // GetAsync
+    public Task<HttpResponseMessage> GetAsync([StringSyntax("Uri")] string? requestUri, CancellationToken? cancellationToken = null) =>
+        SendAsync(new HttpRequestMessage(HttpMethod.Get, requestUri), cancellationToken ?? CancellationToken.None);
+
+    // GetFromJsonAsync
+    public async Task<T?> TryGetFromJsonAsync<T>(string requestUri, JsonSerializerOptions? options = null, CancellationToken cancellationToken = default) {
+        try {
+            return await GetFromJsonAsync<T>(requestUri, options, cancellationToken);
+        }
+        catch (JsonException e) {
+            Console.WriteLine($"Failed to deserialize response from {requestUri}: {e.Message}");
+            return default;
+        }
+        catch (HttpRequestException e) {
+            Console.WriteLine($"Failed to get {requestUri}: {e.Message}");
+            return default;
+        }
+    }
+
+    public async Task<T> GetFromJsonAsync<T>(string requestUri, JsonSerializerOptions? options = null, CancellationToken cancellationToken = default) {
+        options = GetJsonSerializerOptions(options);
+        var request = new HttpRequestMessage(HttpMethod.Get, requestUri);
+        request.Headers.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json"));
+        var response = await SendAsync(request, cancellationToken);
+        response.EnsureSuccessStatusCode();
+        await using var responseStream = await response.Content.ReadAsStreamAsync(cancellationToken);
+
+        return await JsonSerializer.DeserializeAsync<T>(responseStream, options, cancellationToken) ??
+               throw new InvalidOperationException("Failed to deserialize response");
+    }
+
+    // GetStreamAsync
+    public async Task<Stream> GetStreamAsync(string requestUri, CancellationToken cancellationToken = default) {
+        var request = new HttpRequestMessage(HttpMethod.Get, requestUri);
+        request.Headers.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json"));
+        var response = await SendAsync(request, cancellationToken);
+        response.EnsureSuccessStatusCode();
+        return await response.Content.ReadAsStreamAsync(cancellationToken);
+    }
+
+    public async Task<HttpResponseMessage> PutAsJsonAsync<T>([StringSyntax(StringSyntaxAttribute.Uri)] string? requestUri, T value, JsonSerializerOptions? options = null,
+        CancellationToken cancellationToken = default) where T : notnull {
+        options = GetJsonSerializerOptions(options);
+        var request = new HttpRequestMessage(HttpMethod.Put, requestUri);
+        request.Headers.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json"));
+        request.Content = new StringContent(JsonSerializer.Serialize(value, value.GetType(), options),
+            Encoding.UTF8, "application/json");
+        return await SendAsync(request, cancellationToken);
+    }
+
+    public async Task<HttpResponseMessage> PostAsJsonAsync<T>([StringSyntax(StringSyntaxAttribute.Uri)] string? requestUri, T value, JsonSerializerOptions? options = null,
+        CancellationToken cancellationToken = default) where T : notnull {
+        options ??= new JsonSerializerOptions();
+        // options.Converters.Add(new JsonFloatStringConverter());
+        // options.Converters.Add(new JsonDoubleStringConverter());
+        // options.Converters.Add(new JsonDecimalStringConverter());
+        options.DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull;
+        var request = new HttpRequestMessage(HttpMethod.Post, requestUri);
+        request.Headers.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json"));
+        request.Content = new StringContent(JsonSerializer.Serialize(value, value.GetType(), options),
+            Encoding.UTF8, "application/json");
+        return await SendAsync(request, cancellationToken);
+    }
+
+    public async IAsyncEnumerable<T?> GetAsyncEnumerableFromJsonAsync<T>([StringSyntax(StringSyntaxAttribute.Uri)] string? requestUri, JsonSerializerOptions? options = null) {
+        options = GetJsonSerializerOptions(options);
+        var res = await GetAsync(requestUri);
+        options.PropertyNameCaseInsensitive = true;
+        var result = JsonSerializer.DeserializeAsyncEnumerable<T>(await res.Content.ReadAsStreamAsync(), options);
+        await foreach (var resp in result) yield return resp;
+    }
+
+    public static async Task<bool> CheckSuccessStatus(string url) {
+        //cors causes failure, try to catch
+        try {
+            var resp = await Client.GetAsync(url);
+            return resp.IsSuccessStatusCode;
+        }
+        catch (Exception e) {
+            Console.WriteLine($"Failed to check success status: {e.Message}");
+            return false;
+        }
+    }
+
+    public async Task<HttpResponseMessage> PostAsync(string uri, HttpContent? content, CancellationToken cancellationToken = default) {
+        var request = new HttpRequestMessage(HttpMethod.Post, uri) {
+            Content = content
+        };
+        return await SendAsync(request, cancellationToken);
+    }
+
+    public async Task DeleteAsync(string url) {
+        var request = new HttpRequestMessage(HttpMethod.Delete, url);
+        await SendAsync(request);
+    }
+
+    public async Task<HttpResponseMessage> DeleteAsJsonAsync<T>(string url, T payload) {
+        var request = new HttpRequestMessage(HttpMethod.Delete, url) {
+            Content = new StringContent(JsonSerializer.Serialize(payload), Encoding.UTF8, "application/json")
+        };
+        return await SendAsync(request);
+    }
+}
+#endif