Commit: 3ad89fdb643116df3c85c9acf28f884287f504ee - pixivd_tools (hg)

Auxiliary tools for users of pixiv_down

Commit MetaInfo

• Revision: 3ad89fdb643116df3c85c9acf28f884287f504ee (tree)
• Time: 2023-01-29 12:34:57
• Author: nemophila <stigma@disr...>
• Commiter: nemophila

Log Message

Add pixiv_combine_users program

Change Summary

• modified: .hgignore
• modified: dub.sdl
• add: pixiv_combine_users.d

Incremental Difference

diff -r 42f2bc4f33b5 -r 3ad89fdb6431 .hgignore
--- a/.hgignore	Sun Jan 29 13:27:49 2023 +1000
+++ b/.hgignore	Sun Jan 29 13:34:57 2023 +1000
@@ -7,6 +7,7 @@
 
 # Executables
 pixiv_remove_unfollowed
+pixiv_combine_users
 
 # Logs
 .log

diff -r 42f2bc4f33b5 -r 3ad89fdb6431 dub.sdl
--- a/dub.sdl	Sun Jan 29 13:27:49 2023 +1000
+++ b/dub.sdl	Sun Jan 29 13:34:57 2023 +1000
@@ -13,3 +13,9 @@
     targetType "executable"
     targetName "pixiv_remove_unfollowed"
 }
+
+configuration "combine_users" {
+    sourceFiles "pixiv_combine_users.d"
+    targetType "executable"
+    targetName "pixiv_combine_users"
+}

diff -r 42f2bc4f33b5 -r 3ad89fdb6431 pixiv_combine_users.d
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/pixiv_combine_users.d	Sun Jan 29 13:34:57 2023 +1000
@@ -0,0 +1,218 @@
+/**
+ * A program which combines all the directories for each pixiv user ID.
+ *
+ * When pixiv_down downloads an item, it checks if a directory exists in the
+ * form <userID>_<displayName>.  The display name is used to make navigation
+ * familiar -- so you know who you're looking for.  However, this value can
+ * change.  As such, it's possible to end up with duplicates of the same
+ * content.
+ *
+ * This program will let you know of all the options available (i.e. each
+ * <displayName> that has been used) along with the current <displayName>.
+ * It'll then move each file over in to the chosen directory (keeping
+ * modification and create dates).  It will not (by default) overwrite any
+ * existing files.
+ */
+module pixiv_combine_users;
+
+import std.file;
+import std.path : buildPath;
+import std.stdio;
+
+import std.experimental.logger;
+
+import mlib.configparser;
+import mlib.directories;
+import mlib.trash;
+
+import pixivd;
+import pixivd.types;
+
+struct Config
+{
+    string sessid;
+    string baseFolder;
+}
+
+FileLogger logger;
+
+int main(string[] args)
+{
+    Config config;
+    logger = new FileLogger("logs/pixiv_combine_users.log");
+
+    try {
+        loadConfig(config);
+    } catch (FileException fe) {
+        logger.critical(fe.msg);
+        stderr.writeln(fe.msg);
+        stderr.writeln("Have you used both pixivd and pixiv_down before?");
+        return 1;
+    } catch (NoSectionException nse) {
+        stderr.writeln("Couldn't find 'output' section in the pixiv_down config.");
+        stderr.writeln("Please see pixiv_down documentation on setting this.");
+        return 1;
+    } catch (NoOptionException noe) {
+        stderr.writeln("Couldn't find the 'base_folder' option in the pixiv_down config.");
+        stderr.writeln("Please see pixiv_down documentation on setting this.");
+        return 1;
+    }
+
+    string[][string] dupes = findDupes(config);
+    removeDupes(dupes, config);
+    return 0;
+}
+
+/**
+ * Initialize a Config structure.
+ *
+ * This attempts to read both the pixivd PHPSESSID and pixiv_down's settings.
+ * pixivd is used to check the current user's <displayName>, while pixiv_down's
+ * settings are used to see where the pictures are stored.
+ */
+void loadConfig(ref Config config)
+{
+    import std.string : strip;
+
+    DirEntry configDir = open(Directory.config);
+    string pdPath = buildPath(configDir.name, "pixivd", "phpsessid.txt");
+    string pdownPath = buildPath(configDir.name, "pixiv_down", "settings.conf");
+
+    if (false == exists(pdPath)) {
+        throw new FileException(pdPath, "file does not exist");
+    }
+    if (false == exists(pdownPath)) {
+        throw new FileException(pdownPath, "file does not exist");
+    }
+
+    config.sessid = File(pdPath, "r").readln().strip();
+
+    scope ConfigParser parser = new ConfigParser();
+    parser.read(pdownPath);
+    config.baseFolder = parser.get("output", "base_folder");
+}
+
+/**
+ * Find all directories where the pixiv ID is duplicated.
+ *
+ * The returned associative array is in the form:
+ *   "id": [
+ *       "displayName1",
+ *       "displayName2",
+ *       ....
+ *       "displayNameN"
+ *   ]
+ *
+ * Each returned ID will have more than one <displayName>.
+ */
+string[][string] findDupes(in ref Config config)
+{
+    import std.array : join, split;
+    import std.path : baseName;
+
+    const cwd = getcwd();
+    scope(exit) chdir(cwd);
+
+    chdir(config.baseFolder);
+
+    string[][string] ids;
+
+    foreach(string name; dirEntries(config.baseFolder, SpanMode.shallow)) {
+        string bname = baseName(name);
+        string[] spl = bname.split("_");
+        string id = spl[0];
+
+        /* Don't include any non-numerical "ids" (other directories) */
+        if (id[0] < '0' && id[0] > '9')
+            continue;
+
+        ids[id] ~= spl[1..$].join("_");
+    }
+
+    foreach(id, names; ids) {
+        if (names.length == 1) {
+            ids.remove(id);
+        }
+    }
+    
+    return ids;
+}
+
+void prettyPrint(string[] arr)
+{
+    foreach(num, elem; arr) {
+        writefln("   %d) %s", num, elem);
+    }
+}
+
+void removeDupes(string[][string] dupes, in ref Config config)
+{
+    import core.thread : Thread;
+    import core.time : dur;
+    import std.string : empty;
+
+    Client client = new Client(config.sessid);
+    foreach(id, names; dupes) {
+        try {
+            FullUser user = client.fetchUser(id);
+            writefln("All names for ID %s (current: %s):", id, user.userName);
+            prettyPrint(names);
+            write("Which number should be used? ");
+            int choice;
+            readf!" %d"(choice);
+            removeDupe(id, names, choice, config);
+        } catch (PixivJSONException pje) {
+            logger.criticalf("Fetching ID '%s': %s", id, pje.message);
+            stderr.writeln(pje.msg);
+            continue;
+        }
+
+        writeln("Sleeping for 3 seconds");
+        Thread.sleep(dur!"seconds"(3));
+    }
+    writeln("Done removing duplicates! (if any!)");
+}
+
+void removeDupe(string id, string[] names, int choice, in ref Config config)
+{
+    import std.algorithm.mutation : remove;
+    import std.array : array, split;
+    import std.path : baseName, dirSeparator;
+
+    const newPath = buildPath(config.baseFolder, id ~ "_" ~ names[choice]);
+    writefln("Moving all files for %s to %s", names[choice], newPath);
+    names = names.remove(choice);
+
+    /* eww */
+    foreach(name; names) {
+        const baseDir = buildPath(config.baseFolder, id ~ "_" ~ name);
+        foreach(entry; dirEntries(baseDir, SpanMode.breadth)) {
+
+            // We need to account for Manga and multi-page illustrations.
+            string maybeIllustId = entry.name.split(dirSeparator)[$-2];
+            string destPath = newPath;
+            if (maybeIllustId != (id ~ "_" ~ name)) {
+                // Is Manga or multi-paged illustration.
+                destPath = buildPath(destPath, maybeIllustId);
+                logger.tracef("Found Manga or multi-paged illust--output: %s", destPath);
+            }
+
+            if (false == exists(buildPath(destPath, baseName(entry.name)))) {
+                writefln("%s does not exist in %s... Copying.", entry.name, destPath);
+                string fname = baseName(entry.name);
+                destPath = buildPath(destPath, fname);
+                // FIXME: Why not just rename() the directory?
+                if (isDir(entry.name)) {
+                    logger.infof("Making directory: %s", destPath);
+                    mkdir(destPath);
+                    continue;
+                }
+                logger.infof("Copying %s to %s", entry.name, destPath);
+                copy(entry.name, destPath);
+            }
+        }
+        writeln("Moved everything from ", baseDir, "... Deleting.");
+        logger.infof("Trashing '%s'", baseDir);
+        trash(baseDir);
+    }
+}