From d65f0e4fbfe6d9c145291602ac365b09a607ebaf Mon Sep 17 00:00:00 2001
From: gordon <gordon@freeman.free>
Date: Wed, 13 Feb 2019 15:11:32 +0100
Subject: [PATCH] first commit

---
 README.md               |  14 +++++
 archive-download.py     | 116 ++++++++++++++++++++++++++++++++++++++++
 install_dependencies.sh |   2 +
 3 files changed, 132 insertions(+)
 create mode 100644 README.md
 create mode 100755 archive-download.py
 create mode 100755 install_dependencies.sh

diff --git a/README.md b/README.md
new file mode 100644
index 0000000..dc13ad2
--- /dev/null
+++ b/README.md
@@ -0,0 +1,14 @@
+This program runs with python3.
+Required packages are :
+    - BeautifulSoup4
+    - lxml
+    - urllib3
+
+
+This program allows you to download audio files likes concerts etc from the website archive.org.
+Usage in cli is :
+        "./archive-download.py 'url' 'directory'\" with :
+            - url : the url of the files you want to download
+            - directory : a path where you want to folder to be created e.g '~/Downloads'
+
+
diff --git a/archive-download.py b/archive-download.py
new file mode 100755
index 0000000..ef816ab
--- /dev/null
+++ b/archive-download.py
@@ -0,0 +1,116 @@
+#! /usr/bin/env python3
+import urllib3
+import sys, os, re
+import shutil
+import certifi
+
+from bs4 import BeautifulSoup
+#import lxml
+
+urllib3.disable_warnings(urllib3.exceptions.InsecureRequestWarning)
+
+if len(sys.argv) != 3:
+    print("This program allows you to download audio files likes concerts etc " +
+        "from the website archive.org.\nUsage in cli is :\n" +
+        "\"./archive-download.py 'url' 'directory'\" with :\n" +
+        "\t- url : the url of the files you want to download\n" +
+        "\t- directory : a path where you want to folder to be created" +
+        " e.g '~/Downloads'"
+        )    
+    exit()
+
+def getHTML(URL):
+    """
+    Return a html document from a URL link.
+    """
+    http = urllib3.PoolManager(cert_reqs='CERT_REQUIRED',ca_certs=certifi.where())
+    r = http.request('GET',URL)
+    if r == urllib3.exceptions.SSLError:
+        print("Unable to establish connection.")
+        exit()
+    return r.data
+    
+
+def getTracks(URL):
+    """
+    Return a dictionnary with key = track_names and values = track_links.
+    """
+    soup = BeautifulSoup(getHTML(URL), features="lxml")
+    bloc = soup.find("div", {"class":"container container-ia width-max"})
+    raw_names = bloc.find_all("meta", {"itemprop":"name"})
+    raw_links = bloc.find_all("link", {"itemprop":"associatedMedia"})
+    track_names = []
+    track_links = []
+    i = 0
+    for raw_name in raw_names:
+            i += 1
+            if i < 10:
+                nb = "0" + str(i)
+            else:
+                nb = str(i)
+            track_names.append(nb + " - " + raw_name.get('content') + ".mp3")
+    for i in range(len(raw_links)//2):
+        track_links.append(raw_links[i*2].get('href'))
+    tracks = {}
+    for track in range(len(track_names)):
+        tracks[track_names[track]] = track_links[track]
+    return tracks
+
+def getFolderName(URL):
+    """
+    Return a string representing the name of the folder wich will be
+    created to store all tracks.
+    """
+    soup = BeautifulSoup(getHTML(URL), features="lxml")
+    folderName = soup.find("h1", {"class":"sr-only"})
+    folderName = folderName.get_text()
+    folderName = re.sub('\n','',folderName)
+    folderName = re.sub(' +', '_', folderName)
+    folderName = re.sub('^_+','',folderName)
+    folderName = re.sub('_+$','',folderName)
+    
+    return folderName
+    
+    
+def writeTrack(track_name, track_link):
+    """
+    Writes out a track in the current directory.
+    """
+    url = track_link
+    c = urllib3.PoolManager()
+
+    with c.request('GET',url, preload_content=False) as resp, open(track_name.replace("/", "-"), 'wb') as out_file:
+        shutil.copyfileobj(resp, out_file)
+
+
+def writeAll(URL):
+    tracks = getTracks(URL)
+    print("Successfully located track links and names.")
+    folder = getFolderName(URL)
+    os.mkdir(folder) # creates a sub-directory in current directory to store the live
+    os.chdir(folder) # making the created folder the current working folder
+    print("Successfully created a sub-directory to download tracks in.")
+    n = len(tracks)
+    i = 0
+    print("Downloading tracks, this might take a while depending on your connection and the size of your request.")
+    for track_name, track_link in tracks.items():
+        i += 1
+        writeTrack(track_name, track_link)
+        print("Wrote {}/{} tracks".format(i,n)) # to know how the progress    
+    print("All tracks successfully written. Bye")
+
+
+    
+url = sys.argv[1]
+path = sys.argv[2]
+
+os.chdir(os.path.expanduser(path))
+
+writeAll(url)
+           
+#url = "https://archive.org/details/gd72-08-27.sbd.orf.3328.sbeok.shnf/gd1972-08-27d1t01-orf.shn"
+    
+
+
+
+
diff --git a/install_dependencies.sh b/install_dependencies.sh
new file mode 100755
index 0000000..35b37b9
--- /dev/null
+++ b/install_dependencies.sh
@@ -0,0 +1,2 @@
+#!/bin/bash
+pip3 install BeautifulSoup4 lxml urllib3