feat: support .tar.gz and .gz download

build.zig

@@ -11,6 +11,7 @@ pub fn build(b: *std.Build) void {
         .{ .name = "jdz_allocator" },
         .{ .name = "network" },
         .{ .name = "chroma" },
+        .{ .name = "temp" },
         .{
             .name = "euc-jis-2004-zig",
             .module = "euc-jis-2004",

build.zig.zon

@@ -59,6 +59,10 @@
             .url = "git+https://github.com/adia-dev/chroma-zig#c8cd1144aa800f87db3c4bee3d1f0fa0e8c96feb",
             .hash = "12209a8a991121bba3b21f31d275588690dc7c0d7fa9c361fd892e782dd88e0fb2ba",
         },
+        .temp = .{
+            .url = "git+https://github.com/abhinav/temp.zig#6c02e98497b32c78fb67db576dcbc9b1c42c1a04",
+            .hash = "122095027cb551451a12706f958487bacc9bec1dfcd8c6bdacff890fb8d641755a58",
+        },
     },
     .paths = .{
         "build.zig",

conf/toyskkserv.zon

@@ -4,7 +4,8 @@
     .fallback_to_google = true,
     .update_schedule = "monthly",
     .dictionaries = .{
-        "url",
-        "path",
+        .{ .url = "https://someurl" },
+        .{ .url = "https://someurl.gz" },
+        .{ .url = "https://abc.tar.gz", .files = "dir/file_a, dir/file_b" },
     },
 }

src/cmd/server_cmd.zig

@@ -3,7 +3,7 @@ const Server = @import("../server/server.zig").Server;
 const config = @import("../config.zig");
 const jdz_allocator = @import("jdz_allocator");
 const utils = @import("../utils/utils.zig");
-const download = @import("../http/download.zig");
+const dict_location = @import("../dict/dict_location.zig");
 
 pub fn serve() !void {
     var jdz = jdz_allocator.JdzAllocator(.{}).init();
@@ -26,7 +26,6 @@ pub fn serve() !void {
     };
     defer {
         cfg.deinit(allocator);
-        allocator.destroy(cfg);
     }
     const fmt =
         \\Config loaded:
@@ -43,12 +42,11 @@ pub fn serve() !void {
         const download_alloc = gpa.allocator();
 
         utils.log.info("Start downloading missing dictionaries", .{});
-        download.downloadFiles(
+        dict_location.DictLocation.Download.downloadDicts(
             download_alloc,
             cfg.dictionaries,
             cfg.dictionary_directory,
             false,
-            downloadProgress,
         ) catch |err| {
             utils.log.err("Download failed due to {}", .{err});
         };
@@ -73,7 +71,3 @@ pub fn serve() !void {
     };
     utils.log.info("Server exited", .{});
 }
-
-fn downloadProgress(url: []const u8, result: download.Result) void {
-    utils.log.debug("{s} {s}", .{ utils.fs.extractFilename(url), result.toString() });
-}

src/cmd/update_cmd.zig

@@ -1,7 +1,7 @@
 const std = @import("std");
 const config = @import("../config.zig");
 const utils = @import("../utils/utils.zig");
-const download = @import("../http/download.zig");
+const dict_location = @import("../dict/dict_location.zig");
 const log = std.log;
 
 pub fn updateDicts() !void {
@@ -22,21 +22,12 @@ pub fn updateDicts() !void {
     }
 
     log.info("Start updating at {s}", .{cfg.dictionary_directory});
-    download.downloadFiles(
+    dict_location.DictLocation.Download.downloadDicts(
         alloc,
         cfg.dictionaries,
         cfg.dictionary_directory,
         true,
-        downloadProgress,
     ) catch |err| {
         utils.log.err("Download failed due to {}", .{err});
     };
 }
-
-fn downloadProgress(url: []const u8, result: download.Result) void {
-    switch (result) {
-        .Failed => utils.log.err("{s} {s}", .{ utils.fs.extractFilename(url), result.toString() }),
-        .Downloaded => utils.log.info("{s} {s}", .{ utils.fs.extractFilename(url), result.toString() }),
-        else => utils.log.debug("{s} {s}", .{ utils.fs.extractFilename(url), result.toString() }),
-    }
-}

src/config.zig

@@ -1,43 +1,36 @@
 const std = @import("std");
 const zgf = @import("zon_get_fields");
 const utils = @import("utils/utils.zig");
+const DictLocation = @import("dict/dict_location.zig").DictLocation;
 
 const require = @import("protest").require;
 
 pub const Config = struct {
-    dictionary_directory: []const u8,
-    listen_addr: []const u8,
+    dictionary_directory: []const u8 = &.{},
+    listen_addr: []const u8 = &.{},
     fallback_to_google: bool = false,
-    dictionaries: []const []const u8 = undefined,
-    update_schedule: []const u8,
-
-    pub fn init(allocator: std.mem.Allocator, ast: std.zig.Ast) !Config {
-        var cfg = Config{
-            .dictionary_directory = undefined,
-            .listen_addr = undefined,
-            .fallback_to_google = true,
-            .update_schedule = undefined,
-            .dictionaries = undefined,
-        };
+    dictionaries: []DictLocation = &.{},
+    update_schedule: []const u8 = &.{},
+
+    pub fn init(alloc: std.mem.Allocator, ast: std.zig.Ast) !Config {
+        var cfg = Config{};
 
-        cfg.dictionary_directory = try allocator.dupe(u8, zgf.getFieldVal([]const u8, ast, "dictionary_directory") catch "./toyskkserv-cache");
-        cfg.listen_addr = try allocator.dupe(u8, zgf.getFieldVal([]const u8, ast, "listen_addr") catch "127.0.0.1:1178");
-        cfg.update_schedule = try allocator.dupe(u8, zgf.getFieldVal([]const u8, ast, "update_schedule") catch "");
+        cfg.dictionary_directory = try utils.fs.toAbsolutePath(
+            alloc,
+            zgf.getFieldVal([]const u8, ast, "dictionary_directory") catch "./toyskkserv-cache",
+            null,
+        );
+        cfg.listen_addr = try alloc.dupe(u8, zgf.getFieldVal([]const u8, ast, "listen_addr") catch "127.0.0.1:1178");
+        cfg.fallback_to_google = zgf.getFieldVal(bool, ast, "fallback_to_google") catch false;
 
-        var dict_arr = std.ArrayList([]const u8).init(allocator);
+        var dict_arr = std.ArrayList(DictLocation).init(alloc);
         defer dict_arr.deinit();
 
-        const buf = try allocator.alloc(u8, 48);
-        defer allocator.free(buf);
-        var i: i16 = 0;
+        var idx: i16 = 0;
         while (true) {
-            const slice = try std.fmt.bufPrint(buf[0..], "dictionaries[{d}]", .{i});
-            if (zgf.getFieldVal([]const u8, ast, slice)) |dict| {
-                try dict_arr.append(try allocator.dupe(u8, dict));
-            } else |_| {
-                break;
-            }
-            i = i + 1;
+            const location = loadDictLocation(alloc, ast, idx) catch break;
+            try dict_arr.append(location);
+            idx += 1;
         }
         cfg.dictionaries = try dict_arr.toOwnedSlice();
 
@@ -49,12 +42,39 @@ pub const Config = struct {
         allocator.free(self.listen_addr);
         allocator.free(self.update_schedule);
         for (self.dictionaries) |d| {
-            allocator.free(d);
+            allocator.free(d.url);
+            for (d.files) |f| {
+                allocator.free(f);
+            }
+            allocator.free(d.files);
         }
         allocator.free(self.dictionaries);
     }
 };
 
+fn loadDictLocation(alloc: std.mem.Allocator, ast: std.zig.Ast, index: i16) !DictLocation {
+    const buf = try alloc.alloc(u8, 1024);
+    defer alloc.free(buf);
+
+    const key = try std.fmt.bufPrint(buf[0..], "dictionaries[{d}].url", .{index});
+    const url = try zgf.getFieldVal([]const u8, ast, key);
+
+    var file_arr = std.ArrayList([]const u8).init(alloc);
+    defer file_arr.deinit();
+
+    const file_key = try std.fmt.bufPrint(buf[0..], "dictionaries[{d}].files", .{index});
+    if (zgf.getFieldVal([]const u8, ast, file_key)) |files| {
+        var ite = std.mem.splitScalar(u8, files, ',');
+        while (ite.next()) |file| {
+            try file_arr.append(try alloc.dupe(u8, std.mem.trim(u8, file, " ")));
+        }
+    } else |_| {}
+    return .{
+        .url = try alloc.dupe(u8, url),
+        .files = try file_arr.toOwnedSlice(),
+    };
+}
+
 pub fn loadConfig(alloc: std.mem.Allocator) !*Config {
     const config_files = [_][]const u8{
         "./toyskkserv.zon",
@@ -99,11 +119,14 @@ test "config" {
         alloc.destroy(cfg);
     }
 
-    try require.equal("./dict_cache", cfg.dictionary_directory);
+    try require.isTrue(std.mem.endsWith(u8, cfg.dictionary_directory, "dict_cache"));
+    try require.isTrue(std.fs.path.isAbsolute(cfg.dictionary_directory));
     try require.equal("127.0.0.1:1178", cfg.listen_addr);
     try require.isTrue(cfg.fallback_to_google);
-    try require.equal("monthly", cfg.update_schedule);
 
-    try require.equal("url", cfg.dictionaries[0]);
-    try require.equal("path", cfg.dictionaries[1]);
+    try require.equal("https://someurl", cfg.dictionaries[0].url);
+    try require.equal("https://someurl.gz", cfg.dictionaries[1].url);
+    try require.equal("https://abc.tar.gz", cfg.dictionaries[2].url);
+    try require.equal("dir/file_a", cfg.dictionaries[2].files[0]);
+    try require.equal("dir/file_b", cfg.dictionaries[2].files[1]);
 }

src/dict.zig → src/dict/dict.zig

@@ -1,7 +1,9 @@
 const std = @import("std");
 const btree = @import("btree-zig");
-const skk = @import("skk/skk.zig");
-const utils = @import("utils/utils.zig");
+
+const DictLocation = @import("dict_location.zig").DictLocation;
+const skk = @import("../skk/skk.zig");
+const utils = @import("../utils/utils.zig");
 
 const require = @import("protest").require;
 
@@ -25,13 +27,13 @@ pub const DictManager = struct {
         self.allocator.destroy(self.tree);
     }
 
-    pub fn reloadUrls(self: *@This(), urls: []const []const u8, dictionary_path: []const u8) !void {
+    pub fn reloadLocations(self: *@This(), locations: []const DictLocation, dictionary_path: []const u8) !void {
         clearBtree(self.allocator, self.tree);
-        try self.loadUrls(urls, dictionary_path);
+        try self.loadLocations(locations, dictionary_path);
     }
 
-    pub fn loadUrls(self: *@This(), urls: []const []const u8, dictionary_path: []const u8) !void {
-        const files = try utils.url.translateUrlsToFiles(self.allocator, urls, dictionary_path);
+    pub fn loadLocations(self: *@This(), locations: []const DictLocation, dictionary_path: []const u8) !void {
+        const files = try DictLocation.fileList(self.allocator, locations, dictionary_path);
         defer {
             for (files) |f| {
                 self.allocator.free(f);
@@ -117,7 +119,7 @@ pub const DictManager = struct {
 };
 
 fn loadFile(allocator: std.mem.Allocator, tree: *btree.Btree(skk.Entry, void), filename: []const u8) !void {
-    utils.log.debug("Processing file {s}", .{utils.fs.extractFilename(filename)});
+    utils.log.debug("Processing file {s}", .{std.fs.path.basename(filename)});
 
     var line_buf = [_]u8{0} ** 4096;
     var conv_buf = [_]u8{0} ** 4096;
@@ -146,9 +148,13 @@ test "DictManager" {
     var mgr = try DictManager.init(alloc);
     defer mgr.deinit();
 
-    try mgr.loadUrls(&[_][]const u8{
-        url,
-    }, path);
+    const locations: []const DictLocation = &.{
+        .{
+            .url = url,
+            .files = &.{},
+        },
+    };
+    try mgr.loadLocations(locations, path);
 
     try require.equal("", mgr.findCandidate(""));
     try require.equal("/キロ/", mgr.findCandidate("1024"));
@@ -166,10 +172,10 @@ test "DictManager" {
     }
 
     // reload
-    try mgr.reloadUrls(&[_][]const u8{}, path);
+    try mgr.reloadLocations(&[_]DictLocation{}, path);
     try require.equal("", mgr.findCandidate("1024"));
 
-    try mgr.reloadUrls(&[_][]const u8{url}, path);
+    try mgr.reloadLocations(locations, path);
     try require.equal("/キロ/", mgr.findCandidate("1024"));
 }