Fix bugs, add transfer rate monitor, add graceful stop, support single-use URLs in rippers

build.gradle.kts

@@ -5,13 +5,14 @@
 // gradle clean build -PjavacRelease=21
 // gradle clean build -PcustomVersion=1.0.0-10-asdf
 val customVersion = (project.findProperty("customVersion") ?: "") as String
-val javacRelease = (project.findProperty("javacRelease") ?: "17") as String
+val javacRelease = (project.findProperty("javacRelease") ?: "21") as String
 
 plugins {
  id("fr.brouillard.oss.gradle.jgitver") version "0.9.1"
  id("jacoco")
  id("java")
  id("maven-publish")
+ id("com.gradleup.shadow") version "9.0.0-rc1"
 }
 
 repositories {
@@ -23,11 +24,11 @@ dependencies {
  implementation("com.lmax:disruptor:3.4.4")
  implementation("org.java-websocket:Java-WebSocket:1.5.3")
  implementation("org.jsoup:jsoup:1.16.1")
- implementation("org.json:json:20211205")
+ implementation("org.json:json:20231013")
  implementation("com.j2html:j2html:1.6.0")
  implementation("commons-configuration:commons-configuration:1.10")
  implementation("commons-cli:commons-cli:1.5.0")
- implementation("commons-io:commons-io:2.13.0")
+ implementation("commons-io:commons-io:2.14.0")
  implementation("org.apache.httpcomponents:httpclient:4.5.14")
  implementation("org.apache.httpcomponents:httpmime:4.5.14")
  implementation("org.apache.logging.log4j:log4j-api:2.20.0")
@@ -36,13 +37,19 @@ dependencies {
  implementation("org.graalvm.js:js:22.3.2")
  testImplementation(enforcedPlatform("org.junit:junit-bom:5.10.0"))
  testImplementation("org.junit.jupiter:junit-jupiter")
+ testImplementation("org.mockito:mockito-core:5.+")
  testRuntimeOnly("org.junit.platform:junit-platform-launcher")
 }
 
 group = "com.rarchives.ripme"
 version = "1.7.94"
 description = "ripme"
 
+java {
+ sourceCompatibility = JavaVersion.VERSION_21
+ targetCompatibility = JavaVersion.VERSION_21
+}
+
 jacoco {
  toolVersion = "0.8.12"
 }
@@ -80,6 +87,10 @@ tasks.withType<Jar> {
  })
 }
 
+tasks.shadowJar {
+ transform<com.github.jengelman.gradle.plugins.shadow.transformers.Log4j2PluginsCacheFileTransformer>()
+}
+
 publishing {
  publications {
  create<MavenPublication>("maven") {
@@ -105,6 +116,8 @@ tasks.test {
  includeEngines("junit-vintage")
  }
  finalizedBy(tasks.jacocoTestReport) // report is always generated after tests run
+
+ jvmArgs("-javaagent:${classpath.find { it.name.contains("mockito") }?.absolutePath}")
 }
 
 tasks.register<Test>("testAll") {

gradle/wrapper/gradle-wrapper.properties

@@ -1,6 +1,6 @@
 distributionBase=GRADLE_USER_HOME
 distributionPath=wrapper/dists
-distributionUrl=https\://services.gradle.org/distributions/gradle-8.10.2-bin.zip
+distributionUrl=https\://services.gradle.org/distributions/gradle-8.14.3-bin.zip
 networkTimeout=10000
 validateDistributionUrl=true
 zipStoreBase=GRADLE_USER_HOME

src/main/java/com/rarchives/ripme/App.java

@@ -76,6 +76,9 @@ public static void main(String[] args) throws IOException {
  if (GraphicsEnvironment.isHeadless() || args.length > 0) {
  handleArguments(args);
  } else {
+ // Antialiasing hint, especially for Linux
+ System.setProperty("awt.useSystemAAFontSettings", "on");
+
  if (SystemUtils.IS_OS_MAC_OSX) {
  System.setProperty("apple.laf.useScreenMenuBar", "true");
  System.setProperty("com.apple.mrj.application.apple.menu.about.name", "RipMe");

src/main/java/com/rarchives/ripme/ripper/AbstractHTMLRipper.java

@@ -13,11 +13,7 @@
 import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.nio.file.StandardOpenOption;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
+import java.util.*;
 
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.Logger;
@@ -36,9 +32,6 @@ public abstract class AbstractHTMLRipper extends AbstractRipper {
 
  private static final Logger logger = LogManager.getLogger(AbstractHTMLRipper.class);
 
- private final Map<URL, File> itemsPending = Collections.synchronizedMap(new HashMap<>());
- private final Map<URL, Path> itemsCompleted = Collections.synchronizedMap(new HashMap<>());
- private final Map<URL, String> itemsErrored = Collections.synchronizedMap(new HashMap<>());
  Document cachedFirstPage;
 
  protected AbstractHTMLRipper(URL url) throws IOException {
@@ -76,10 +69,6 @@ protected List<String> getDescriptionsFromPage(Document doc) throws IOException
 
  protected abstract void downloadURL(URL url, int index);
 
- protected DownloadThreadPool getThreadPool() {
- return null;
- }
-
  protected boolean keepSortOrder() {
  return true;
  }
@@ -121,7 +110,7 @@ protected boolean pageContainsAlbums(URL url) {
 
  @Override
  public void rip() throws IOException, URISyntaxException {
- int index = 0;
+ int imageIndex = 0;
  int textindex = 0;
  logger.info("Retrieving " + this.url);
  sendUpdate(STATUS.LOADING_RESOURCE, this.url.toExternalForm());
@@ -176,9 +165,10 @@ public void rip() throws IOException, URISyntaxException {
  }
 
  for (String imageURL : imageURLs) {
- index += 1;
- logger.debug("Found image url #" + index + ": '" + imageURL + "'");
- downloadURL(new URI(imageURL).toURL(), index);
+ imageIndex += 1;
+ logger.debug("Found image url #{} of album {}: {}", imageIndex, this.url, imageURL);
+ setItemsTotal(Math.max(getItemsTotal(), imageIndex));
+ downloadURL(new URI(imageURL).toURL(), imageIndex);
  if (isStopped() || isThisATest()) {
  break;
  }
@@ -206,7 +196,7 @@ public void rip() throws IOException, URISyntaxException {
  workingDir.getCanonicalPath()
  + ""
  + File.separator
- + getPrefix(index)
+ + getPrefix(imageIndex)
  + (tempDesc.length > 1 ? tempDesc[1] : filename)
  + ".txt").exists();
 
@@ -236,12 +226,17 @@ public void rip() throws IOException, URISyntaxException {
  }
  }
 
- // If they're using a thread pool, wait for it.
- if (getThreadPool() != null) {
- logger.debug("Waiting for threadpool " + getThreadPool().getClass().getName());
- getThreadPool().waitForThreads();
+ logger.info("All items queued; total items: {}; url: {}", imageIndex, url);
+
+ // Final total item count is now known
+ setItemsTotal(imageIndex);
+
+ if (getCrawlerThreadPool() != null) {
+ logger.debug("Waiting for crawler threadpool: {}", url);
+ getCrawlerThreadPool().waitForThreads(imageIndex, shouldStop, url);
  }
- waitForThreads();
+
+ waitForRipperThreads();
  }
 
  /**
@@ -338,68 +333,6 @@ protected boolean allowDuplicates() {
  return false;
  }
 
- @Override
- /*
- Returns total amount of files attempted.
- */
- public int getCount() {
- return itemsCompleted.size() + itemsErrored.size();
- }
-
- @Override
- /*
- Queues multiple URLs of single images to download from a single Album URL
- */
- public boolean addURLToDownload(URL url, Path saveAs, String referrer, Map<String,String> cookies, Boolean getFileExtFromMIME) {
- // Only download one file if this is a test.
- if (isThisATest() && (itemsCompleted.size() > 0 || itemsErrored.size() > 0)) {
- stop();
- itemsPending.clear();
- return false;
- }
- if (!allowDuplicates()
- && ( itemsPending.containsKey(url)
- || itemsCompleted.containsKey(url)
- || itemsErrored.containsKey(url) )) {
- // Item is already downloaded/downloading, skip it.
- logger.info("[!] Skipping " + url + " -- already attempted: " + Utils.removeCWD(saveAs));
- return false;
- }
- if (shouldIgnoreURL(url)) {
- sendUpdate(STATUS.DOWNLOAD_SKIP, "Skipping " + url.toExternalForm() + " - ignored extension");
- return false;
- }
- if (Utils.getConfigBoolean("urls_only.save", false)) {
- // Output URL to file
- Path urlFile = Paths.get(this.workingDir + "/urls.txt");
- String text = url.toExternalForm() + System.lineSeparator();
- try {
- Files.write(urlFile, text.getBytes(StandardCharsets.UTF_8), StandardOpenOption.CREATE, StandardOpenOption.APPEND);
- itemsCompleted.put(url, urlFile);
- } catch (IOException e) {
- logger.error("Error while writing to " + urlFile, e);
- }
- }
- else {
- itemsPending.put(url, saveAs.toFile());
- DownloadFileThread dft = new DownloadFileThread(url, saveAs.toFile(), this, getFileExtFromMIME);
- if (referrer != null) {
- dft.setReferrer(referrer);
- }
- if (cookies != null) {
- dft.setCookies(cookies);
- }
- threadPool.addThread(dft);
- }
-
- return true;
- }
-
- @Override
- public boolean addURLToDownload(URL url, Path saveAs) {
- return addURLToDownload(url, saveAs, null, null, false);
- }
-
  /**
  * Queues image to be downloaded and saved.
  * Uses filename from URL to decide filename.
@@ -413,72 +346,6 @@ protected boolean addURLToDownload(URL url) {
  return addURLToDownload(url, "", "");
  }
 
- @Override
- /*
- Cleans up & tells user about successful download
- */
- public void downloadCompleted(URL url, Path saveAs) {
- if (observer == null) {
- return;
- }
- try {
- String path = Utils.removeCWD(saveAs);
- RipStatusMessage msg = new RipStatusMessage(STATUS.DOWNLOAD_COMPLETE, path);
- itemsPending.remove(url);
- itemsCompleted.put(url, saveAs);
- observer.update(this, msg);
-
- checkIfComplete();
- } catch (Exception e) {
- logger.error("Exception while updating observer: ", e);
- }
- }
-
- @Override
- /*
- * Cleans up & tells user about failed download.
- */
- public void downloadErrored(URL url, String reason) {
- if (observer == null) {
- return;
- }
- itemsPending.remove(url);
- itemsErrored.put(url, reason);
- observer.update(this, new RipStatusMessage(STATUS.DOWNLOAD_ERRORED, url + " : " + reason));
-
- checkIfComplete();
- }
-
- @Override
- /*
- Tells user that a single file in the album they wish to download has
- already been downloaded in the past.
- */
- public void downloadExists(URL url, Path file) {
- if (observer == null) {
- return;
- }
-
- itemsPending.remove(url);
- itemsCompleted.put(url, file);
- observer.update(this, new RipStatusMessage(STATUS.DOWNLOAD_WARN, url + " already saved as " + file));
-
- checkIfComplete();
- }
-
- /**
- * Notifies observers and updates state if all files have been ripped.
- */
- @Override
- protected void checkIfComplete() {
- if (observer == null) {
- return;
- }
- if (itemsPending.isEmpty()) {
- super.checkIfComplete();
- }
- }
-
  /**
  * Sets directory to save all ripped files to.
  * @param url
@@ -515,22 +382,20 @@ public void setWorkingDir(URL url) throws IOException, URISyntaxException {
  */
  @Override
  public int getCompletionPercentage() {
- double total = itemsPending.size() + itemsErrored.size() + itemsCompleted.size();
- return (int) (100 * ( (total - itemsPending.size()) / total));
+ double total = getTotalCount();
+ if (total == 0) {
+ return 0;
+ }
+ return (int) (100 * ( (itemsCompleted.size() + itemsErrored.size()) / total));
  }
 
- /**
- * @return
- * Human-readable information on the status of the current rip.
- */
  @Override
- public String getStatusText() {
- return getCompletionPercentage() +
-  "% " +
-  "- Pending: " + itemsPending.size() +
-  ", Completed: " + itemsCompleted.size() +
-  ", Errored: " + itemsErrored.size();
+ public int getPendingCount() {
+ DownloadThreadPool threadPool = getRipperThreadPool();
+ if (threadPool != null) {
+ return threadPool.getPendingThreadCount();
+ }
+ return itemsPending.size();
  }
 
-
 }