A CLI tool for downloading from pixiv.net
Revision | cefcbc2e30803dd5933ae4555346e1247ec505a0 (tree) |
---|---|
Time | 2023-10-29 17:08:40 |
Author | mio <stigma@disr...> |
Commiter | mio |
Replace global 'log' with stdThreadLocalLog
@@ -144,8 +144,6 @@ enum ExitCode | ||
144 | 144 | unknownOptionError = 3, |
145 | 145 | } |
146 | 146 | |
147 | -FileLogger log; | |
148 | - | |
149 | 147 | version(appimage) { |
150 | 148 | enum ProgramName = "pixiv_down"; |
151 | 149 | } |
@@ -308,6 +306,7 @@ void downloadIllust(ref Illustration illust, ref Config conf, | ||
308 | 306 | { |
309 | 307 | import std.algorithm.searching : canFind; |
310 | 308 | import std.array : join, split; |
309 | + import std.experimental.logger : tracef, warningf; | |
311 | 310 | import std.file : FileException, mkdirRecurse, exists; |
312 | 311 | import std.path : buildPath; |
313 | 312 | import std.stdio : stderr; |
@@ -321,7 +320,7 @@ void downloadIllust(ref Illustration illust, ref Config conf, | ||
321 | 320 | |
322 | 321 | stderr.writefln("[%s] Downloading %s - %s", illust.type, illust.userName, |
323 | 322 | illust.title); |
324 | - log.tracef("Downloading post type %s", illust.type); | |
323 | + tracef("Post Type = %s", illust.type); | |
325 | 324 | |
326 | 325 | scope monitor = new ProgressMonitor(illust.pages); |
327 | 326 | conf.client.connect(&monitor.alreadyComplete); |
@@ -331,8 +330,8 @@ void downloadIllust(ref Illustration illust, ref Config conf, | ||
331 | 330 | try { |
332 | 331 | conf.client.downloadIllust(illust, subDirectory, overwrite); |
333 | 332 | } catch (FileException fe) { |
334 | - log.warningf("FileException: %s (overwrite = %s)", fe.msg, | |
335 | - overwrite ? "true" : "false"); | |
333 | + warningf("FileException: %s (overwrite = %s)", fe.msg, | |
334 | + overwrite ? "true" : "false"); | |
336 | 335 | stderr.write("\r"); |
337 | 336 | stderr.writefln("[%s] %s - %s has already been downloaded", illust.type, |
338 | 337 | illust.userName, illust.title); |
@@ -344,15 +343,14 @@ void downloadIllust(ref Illustration illust, ref Config conf, | ||
344 | 343 | void downloadArtist(string id, ContentType type, int offset, |
345 | 344 | ref Config config) |
346 | 345 | { |
346 | + import std.experimental.logger : infof, errorf; | |
347 | 347 | import std.net.curl : CurlException; |
348 | 348 | import std.stdio : stderr; |
349 | 349 | |
350 | 350 | UserBrief up = config.client.fetchUserAll(id); |
351 | 351 | |
352 | 352 | if (ContentType.illustration == type) { |
353 | - | |
354 | - log.infof("Total number of illustrations to download: %d", | |
355 | - up.illusts.length); | |
353 | + infof("Number of Illustrations to download = %d", up.illusts.length); | |
356 | 354 | |
357 | 355 | foreach (elemNum, illustId; up.illusts) { |
358 | 356 | Illustration illust = config.client.fetchIllustration(illustId); |
@@ -360,7 +358,7 @@ void downloadArtist(string id, ContentType type, int offset, | ||
360 | 358 | try { |
361 | 359 | downloadIllust(illust, config); |
362 | 360 | } catch (CurlException ce) { |
363 | - log.warningf("CurlException: %s", ce.msg); | |
361 | + errorf("CurlException: %s", ce.msg); | |
364 | 362 | stderr.writefln("ERROR: Failed to download illust"); |
365 | 363 | stderr.writefln("TRYING AGAIN"); |
366 | 364 | sleep(20, 30); |
@@ -373,8 +371,7 @@ void downloadArtist(string id, ContentType type, int offset, | ||
373 | 371 | sleep(4, 10); |
374 | 372 | } |
375 | 373 | } else if (ContentType.manga == type) { |
376 | - | |
377 | - log.infof("Total number of Manga to download: %d", up.manga.length); | |
374 | + infof("Number of Manga to download: %d", up.manga.length); | |
378 | 375 | |
379 | 376 | foreach (elemNum, illustId; up.manga) { |
380 | 377 | Illustration illust = config.client.fetchIllustration(illustId); |
@@ -382,7 +379,7 @@ void downloadArtist(string id, ContentType type, int offset, | ||
382 | 379 | try { |
383 | 380 | downloadIllust(illust, config); |
384 | 381 | } catch (CurlException ce) { |
385 | - log.warningf("CurlException: %s", ce.msg); | |
382 | + errorf("CurlException: %s", ce.msg); | |
386 | 383 | stderr.writefln("ERROR: Failed to download manga."); |
387 | 384 | stderr.writefln("TRYING AGAIN"); |
388 | 385 | sleep(20, 30); |
@@ -405,6 +402,8 @@ void downloadArtist(string id, ContentType type, int offset, | ||
405 | 402 | void downloadDaily(const ref SysTime begin, const ref SysTime end, |
406 | 403 | string restrict, ref Config conf) |
407 | 404 | { |
405 | + import std.experimental.logger : info, tracef; | |
406 | + | |
408 | 407 | Thumbnail[] dailyIllusts = conf.client.fetchIllustrationsFollowing(); |
409 | 408 | |
410 | 409 | bool reachedEndDate = false; |
@@ -422,7 +421,7 @@ lDownload: | ||
422 | 421 | /* Only download posts uploaded before or on `end`. */ |
423 | 422 | if (end > createDate) { |
424 | 423 | reachedEndDate = true; |
425 | - log.info("reachedEndDate = true"); | |
424 | + info("reachedEndDate = true"); | |
426 | 425 | break; |
427 | 426 | } |
428 | 427 |
@@ -431,13 +430,13 @@ lDownload: | ||
431 | 430 | } |
432 | 431 | |
433 | 432 | if (false == reachedEndDate) { |
434 | - log.logf("finished downloading page %d", page); | |
435 | - log.info("reachedEndDate = false"); | |
436 | - sleep(15, 20); | |
437 | - page += 1; | |
438 | - log.infof("will now download page %d", page); | |
439 | - dailyIllusts = conf.client.fetchIllustrationsFollowing(page); | |
440 | - goto lDownload; | |
433 | + tracef("Finished downloading page %d", page); | |
434 | + info("reachedEndDate = false"); | |
435 | + sleep(15, 20); | |
436 | + page += 1; | |
437 | + tracef("Proceeding to download page %d", page); | |
438 | + dailyIllusts = conf.client.fetchIllustrationsFollowing(page); | |
439 | + goto lDownload; | |
441 | 440 | } |
442 | 441 | } |
443 | 442 |
@@ -452,6 +451,7 @@ lDownload: | ||
452 | 451 | int artistHandle(string[] args, ref Config config) |
453 | 452 | { |
454 | 453 | import std.algorithm.searching : all; |
454 | + import std.experimental.logger : errorf; | |
455 | 455 | import std.stdio : stderr; |
456 | 456 | |
457 | 457 | string enteredType; |
@@ -486,7 +486,7 @@ int artistHandle(string[] args, ref Config config) | ||
486 | 486 | if (all!"(a >= '0' && a <= '9')"(arg)) { |
487 | 487 | ids ~= arg; |
488 | 488 | } else { |
489 | - log.errorf(`Invalid artist ID "%s"`, args); | |
489 | + errorf(`Invalid artist ID "%s"`, args); | |
490 | 490 | |
491 | 491 | stderr.writefln("%s: Invalid artist ID provided.", args[0]); |
492 | 492 | stderr.writeln("Artist IDs are required to be numbers."); |
@@ -532,6 +532,7 @@ int artistHandle(string[] args, ref Config config) | ||
532 | 532 | int artworkHandle(string[] args, ref Config config) |
533 | 533 | { |
534 | 534 | import std.conv : ConvException, to; |
535 | + import std.experimental.logger : info, errorf; | |
535 | 536 | import std.stdio : stderr; |
536 | 537 | |
537 | 538 | // When an exception occurrs. |
@@ -556,14 +557,14 @@ int artworkHandle(string[] args, ref Config config) | ||
556 | 557 | } |
557 | 558 | if ("--group-errors" == arg) { |
558 | 559 | groupErrors = true; |
559 | - log.info("errors will be grouped"); | |
560 | + info("errors will be grouped"); | |
560 | 561 | continue; |
561 | 562 | } |
562 | 563 | |
563 | 564 | try { |
564 | 565 | id = to!size_t(arg); |
565 | 566 | } catch (ConvException ce) { |
566 | - log.errorf("Could not convert %s to size_t", arg); | |
567 | + errorf("Could not convert %s to size_t", arg); | |
567 | 568 | |
568 | 569 | if (groupErrors) { |
569 | 570 | invalidArtworks ~= arg; |
@@ -611,6 +612,7 @@ int dailyHandle(string[] args, ref Config config) | ||
611 | 612 | import core.time : TimeException; |
612 | 613 | import std.algorithm.searching : canFind; |
613 | 614 | import std.datetime.systime : Clock; |
615 | + import std.experimental.logger : infof; | |
614 | 616 | import std.stdio : stderr; |
615 | 617 | |
616 | 618 | string restrict = "both"; |
@@ -682,8 +684,7 @@ int dailyHandle(string[] args, ref Config config) | ||
682 | 684 | return 1; |
683 | 685 | } |
684 | 686 | |
685 | - log.infof(`PARAMS=(restrict=%s rawBeginDate=%s rawEndDate=%s)`, restrict, | |
686 | - rawBeginDate, rawEndDate); | |
687 | + infof("rawBeginDate=%s rawEndDate=%s", rawBeginDate, rawEndDate); | |
687 | 688 | |
688 | 689 | beginDate = Clock.currTime; |
689 | 690 |
@@ -742,6 +743,7 @@ int followingHandle(string[] args, ref Config config) | ||
742 | 743 | { |
743 | 744 | import std.container : SList; |
744 | 745 | import std.conv : to; |
746 | + import std.experimental.logger : infof, tracef; | |
745 | 747 | import std.stdio : stderr, writefln; |
746 | 748 | |
747 | 749 | bool pub = false; |
@@ -813,7 +815,7 @@ int followingHandle(string[] args, ref Config config) | ||
813 | 815 | // Web API uses "hide" and "show" |
814 | 816 | string rest = prv ? "hide" : "show"; |
815 | 817 | |
816 | - log.tracef("rest = %s, skip = %d", rest, skip); | |
818 | + infof("rest=%s skip=%d", rest, skip); | |
817 | 819 | |
818 | 820 | // Fetch all of the accounts to download. |
819 | 821 | // |
@@ -826,7 +828,7 @@ int followingHandle(string[] args, ref Config config) | ||
826 | 828 | writefln("[following]: Downloading API JSON for page %d", pageCount); |
827 | 829 | User[] users = config.client.fetchFollowing(offset, totalNumberOfAccounts, |
828 | 830 | pageLimit, rest); |
829 | - log.tracef("page %d returned %d users.", pageCount, users.length); | |
831 | + infof("Page %d returned %d users.", pageCount, users.length); | |
830 | 832 | offset += users.length; |
831 | 833 | |
832 | 834 | auto allUsers = SList!User(); |
@@ -834,14 +836,14 @@ int followingHandle(string[] args, ref Config config) | ||
834 | 836 | while (offset < totalNumberOfAccounts) { |
835 | 837 | writefln("[following]: Downloading API JSON for page %d", ++pageCount); |
836 | 838 | users = config.client.fetchFollowing(offset, pageLimit, rest); |
837 | - log.tracef("page %d returned %d users.", pageCount, users.length); | |
839 | + infof("Page %d returned %d users.", pageCount, users.length); | |
838 | 840 | allUsers.insertAfter(allUsers[], users); |
839 | 841 | |
840 | 842 | // Increase offset for next page fetch |
841 | 843 | offset += users.length; |
842 | 844 | } |
843 | 845 | |
844 | - log.infof("totalNumberOfAccounts = %d", totalNumberOfAccounts); | |
846 | + infof("totalNumberOfAccounts = %d", totalNumberOfAccounts); | |
845 | 847 | |
846 | 848 | size_t artistIndex = skip + 1; |
847 | 849 | foreach(User user; allUsers) { |
@@ -853,7 +855,7 @@ int followingHandle(string[] args, ref Config config) | ||
853 | 855 | |
854 | 856 | writefln("[following]: Downloading artist %u of %u", artistIndex, |
855 | 857 | totalNumberOfAccounts); |
856 | - log.tracef("user %u/%u (ID = %s)", artistIndex, totalNumberOfAccounts, | |
858 | + tracef("user %u/%u (ID = %s)", artistIndex, totalNumberOfAccounts, | |
857 | 859 | user.userId); |
858 | 860 | artistHandle([args[0], "artist", user.userId], config); |
859 | 861 | artistIndex += 1; |
@@ -1132,14 +1134,14 @@ version(appimage) { | ||
1132 | 1134 | void |
1133 | 1135 | logLibGMVersion() |
1134 | 1136 | { |
1137 | + import std.experimental.logger : infof; | |
1135 | 1138 | import std.string : fromStringz; |
1136 | 1139 | import graphicsmagick_c.magick.version_ : GetMagickVersion, |
1137 | 1140 | MagickLibVersionText, MagickQuantumDepth; |
1138 | 1141 | |
1139 | - log.infof("GraphicsMagick Version D: %s", | |
1140 | - fromStringz(GetMagickVersion(null))); | |
1141 | - log.infof("GraphicsMagick Version S: %s", MagickLibVersionText); | |
1142 | - log.infof("GraphicsMagick Depth S: %s", MagickQuantumDepth); | |
1142 | + infof("GraphicsMagick Version D: %s", fromStringz(GetMagickVersion(null))); | |
1143 | + infof("GraphicsMagick Version S: %s", MagickLibVersionText); | |
1144 | + infof("GraphicsMagick Depth S: %s", MagickQuantumDepth); | |
1143 | 1145 | } |
1144 | 1146 | |
1145 | 1147 | void |
@@ -1164,6 +1166,7 @@ resetSessionID() | ||
1164 | 1166 | Config |
1165 | 1167 | loadConfig() |
1166 | 1168 | { |
1169 | + import std.experimental.logger : infof; | |
1167 | 1170 | import std.file : exists, mkdirRecurse; |
1168 | 1171 | import std.path : buildPath; |
1169 | 1172 | import std.string : strip; |
@@ -1178,8 +1181,7 @@ loadConfig() | ||
1178 | 1181 | ProjectDirectories dirs = getProjectDirectories(null, "YumeNeru Software", |
1179 | 1182 | "pixiv_down"); |
1180 | 1183 | UserDirectories userDirs = getUserDirectories(); |
1181 | - log.infof("dirs.configDir = %s", dirs.configDir); | |
1182 | - log.infof("userDirs.pictureDir = %s", userDirs.pictureDir); | |
1184 | + infof("Configuration Directory = %s", dirs.configDir); | |
1183 | 1185 | |
1184 | 1186 | if (exists(dirs.configDir)) { |
1185 | 1187 | string configFilePath = buildPath(dirs.configDir, "settings.conf"); |
@@ -1195,8 +1197,8 @@ loadConfig() | ||
1195 | 1197 | // Set default values |
1196 | 1198 | config.baseFolder = userDirs.pictureDir; |
1197 | 1199 | } |
1198 | - log.infof("config.baseFolder = %s", config.baseFolder); | |
1199 | - log.infof("data directory = %s", dirs.dataDir); | |
1200 | + infof("Output Directory = %s", config.baseFolder); | |
1201 | + infof("Local Data Directory = %s", dirs.dataDir); | |
1200 | 1202 | |
1201 | 1203 | const idFilePath = buildPath(dirs.dataDir, ".phpsessid"); |
1202 | 1204 |
@@ -1234,6 +1236,8 @@ int | ||
1234 | 1236 | main(string[] args) |
1235 | 1237 | { |
1236 | 1238 | import std.stdio : stderr, writeln, writefln; |
1239 | + import std.experimental.logger : stdThreadLocalLog; | |
1240 | + import logger; | |
1237 | 1241 | |
1238 | 1242 | version(appimage) { |
1239 | 1243 | args[0] = ProgramName; |
@@ -1286,9 +1290,7 @@ version(appimage) { | ||
1286 | 1290 | return 2; |
1287 | 1291 | } |
1288 | 1292 | |
1289 | - log = new FileLogger("pixiv_down.log"); | |
1290 | - // By default, FileLogger opens in append. | |
1291 | - log.file.reopen("pixiv_down.log", "w"); | |
1293 | + stdThreadLocalLog = new PixivLogger("pixiv_down.log"); | |
1292 | 1294 | |
1293 | 1295 | version (GMagick_Dynamic) { |
1294 | 1296 | bool loadedGM = loadLibGM(args[0]); |
@@ -0,0 +1,30 @@ | ||
1 | +/* | |
2 | + * pixiv_down - CLI-based downloading tool for https://www.pixiv.net. | |
3 | + * Copyright (C) 2023 Mio | |
4 | + * | |
5 | + * This program is free software: you can redistribute it and/or modify | |
6 | + * it under the terms of the GNU General Public License as published by | |
7 | + * the Free Software Foundation, version 3 of the License. | |
8 | + * | |
9 | + * This program is distributed in the hope that it will be useful, | |
10 | + * but WITHOUT ANY WARRANTY; without even the implied warranty of | |
11 | + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the | |
12 | + * GNU General Public License for more details. | |
13 | + * | |
14 | + * You should have received a copy of the GNU General Public License | |
15 | + * along with this program. If not, see <https://www.gnu.org/licenses/>. | |
16 | + */ | |
17 | +module logger; | |
18 | + | |
19 | +import std.experimental.logger : FileLogger; | |
20 | + | |
21 | +public class PixivLogger : FileLogger | |
22 | +{ | |
23 | +private: | |
24 | + | |
25 | + public this(string logFile) @safe | |
26 | + { | |
27 | + super(logFile); | |
28 | + this.file.reopen(logFile, "w+"); | |
29 | + } | |
30 | +} |