rss: implement parallel downloads

diff --git a/man/rss.1.scd b/man/rss.1.scd
index 9ea9f7ca3afac6ce4e412f32eca543e041599082..ba58ab28ee17b596d43af4a43361f459bd02c83f 100644 (file)
--- a/man/rss.1.scd
+++ b/man/rss.1.scd
@@ -4,11 +4,13 @@ rss(1)
 rss - fetch rss feeds and launch sfeed_curses
 
 # SYNOPSIS
 rss - fetch rss feeds and launch sfeed_curses
 
 # SYNOPSIS

-rss [path to feeds]
+rss [path to feeds] [number of threads]

 
 # DESCRIPTION
 
 # DESCRIPTION

-Downloads feeds from the first argument (defaults to ~/org/feeds), and opens
-them in sfeed_curses. Depends on sfeed_curses and libxml2.
+Downloads RSS feeds listed in the file provided by the first argument (defaults
+to ~/org/feeds), and opens them in sfeed_curses. Download with as many parallel
+jobs as specified in the second argument (defaults to 4). Depends on
+sfeed_curses and libxml2.

 
 # COPYRIGHT
 This is free and unencumbered software released into the public domain. For more
 
 # COPYRIGHT
 This is free and unencumbered software released into the public domain. For more

diff --git a/rss b/rss
index cfd93e6d5163e9d5f1e77cd910e4da200effbf0c..5a1da2de44a451ff818fdc1eef0a0f52c59c6085 100755 (executable)
--- a/rss
+++ b/rss
@@ -1,24 +1,33 @@
 #!/usr/bin/env sh
 
 #!/usr/bin/env sh
 

-file=${2:-$HOME/org/feeds}
-total=$(wc -l "$file" | cut -d " " -f 1)
-cachedir=${XDG_CACHE_HOME:-$HOME/.cache/}/sfeed
+dl() {
+       while read -r feed; do
+               printf "Processing %s\n" "$feed"
+               curl -s "$feed" | xmllint --format - 1> /tmp/sfeed 2> /dev/null
+               
+               title="$(grep -m 1 title /tmp/sfeed | \
+                       xmllint --xpath "string(//title)" - 2> /dev/null)"
+               domain=$(echo "$feed" | cut -d/ -f 3)
+               fname=$(echo "${title:-$domain}" | sed 's|/|!|g' | head -c 25)
+               sfeed < /tmp/sfeed > "$cachedir/$fname"
+       done < "$1"
+}

 
 

-mkdir -p "$cachedir"
+trap 'trap - SIGTERM && kill -- -$$' INT TERM EXIT

 
 

-i=1
-while read -r feed; do
-       printf "Downloading %s\n%s/%s\r" "$feed" "$i" "$total"
-       curl -s "$feed" | xmllint --format - > /tmp/sfeed
-       
-       title="$(grep -m 1 title /tmp/sfeed | xmllint --xpath "string(//title)" -)"
-       domain=$(echo "$feed" | cut -d/ -f 3)
-       fname=$(echo "${title:-$domain}" | sed 's|/|!|g' | head -c 25)
-       sfeed < /tmp/sfeed > "$cachedir/$fname"
+file=${1:-$HOME/org/feeds}
+total=$(wc -l "$file" | cut -d " " -f 1)
+cachedir=${XDG_CACHE_HOME:-$HOME/.cache/}/sfeed
+threads=${2:-4}
+pere=$(($((total + threads - 1)) / threads))

 
 

-       i=$((i+1))
-done < "$file"
+mkdir -p "$cachedir/split"
+(cd "$cachedir/split" || exit; split -l "$pere" "$file")

 
 

-rm /tmp/sfeed
+for j in "$cachedir/split"/*; do
+       dl "$j" &
+done

 
 

+wait
+rm -rf /tmp/sfeed "$cachedir/split"

 sfeed_curses "$cachedir"/*
 sfeed_curses "$cachedir"/*