From 4a8d849c0ae2e2b24dde90028fc95697040af898 Mon Sep 17 00:00:00 2001 From: Ray Lyon <36998292+skoobasteeve@users.noreply.github.com> Date: Sat, 28 Nov 2020 16:53:09 -0500 Subject: [PATCH 01/24] fix stupid merge problem, add functions --- ffmpeg/ffmpeg-batch-encode.sh | 98 +++++++++++++++++++++++++++-------- 1 file changed, 75 insertions(+), 23 deletions(-) diff --git a/ffmpeg/ffmpeg-batch-encode.sh b/ffmpeg/ffmpeg-batch-encode.sh index 951b804..14e3cc5 100755 --- a/ffmpeg/ffmpeg-batch-encode.sh +++ b/ffmpeg/ffmpeg-batch-encode.sh @@ -1,10 +1,20 @@ #!/bin/bash -# Help function +### VARIABLES ### + +PROGNAME="$(basename "$0")" +INPUT_SOURCE=$1 +QUALITY_HD=23 +QUALITY_4K=22 +FILEDIR=$(dirname "$INPUT_SOURCE") + + +### FUNCTIONS### + +### Function: Help Help() { - # Display Help echo "This script uses sensible ffmpeg options to batch encode MKV files in a directory to compressed H264 MKVs." echo "You can change the CRF parameters in the script, defaults are 24 for HD and 22 for 4K." echo @@ -14,35 +24,77 @@ Help() echo "Learn more about FFmpeg's quality settings: https://trac.ffmpeg.org/wiki/Encode/H.264" } -### Script ### +### Funtion: Error -DIRECTORY=$1 -QUALITY_HD=22 -QUALITY_4K=24 +error_exit() +{ + echo "${PROGNAME}: ${1:-"Unknown Error"}" 1>&2 + exit 1 +} -# Basic error handling + +# Function: encode each file in the directory with different CRF setting based on resolution + +folder_encode () { + if [ ! -d "$INPUT_SOURCE/output" ]; then + mkdir "$INPUT_SOURCE/output" + fi + + for FILE in "$INPUT_SOURCE"/*.*; do + RES=$(ffprobe -v error -select_streams v:0 -show_entries stream=width -of default=noprint_wrappers=1:nokey=1 "$FILE") + FILENAME=$(basename "$FILE") + if [[ $RES -gt 1920 ]]; then + echo "File is 4K or higher, encoding using CRF $QUALITY_4K" + ffmpeg -i "$FILE" -c:v libx264 -preset slow -tune film -crf "$QUALITY_4K" -c:a copy "$INPUT_SOURCE"/output/"$FILENAME" || echo "ERROR Line $LINENO: File not encoded, unknown error occurred." 1>&2 + elif [[ $RES -le 1920 ]] && [[ -n $RES ]]; then + echo "File is HD or lower, encoding using CRF $QUALITY_HD" + ffmpeg -i "$FILE" -c:v libx264 -preset slow -tune film -crf "$QUALITY_HD" -c:a copy "$INPUT_SOURCE"/output/"$FILENAME" || echo "ERROR Line $LINENO: File not encoded, unknown error occurred." 1>&2 + else + echo "ERROR Line $LINENO: Source file $FILE is not a valid video file" 1>&2 + echo "Skipping..." + fi + done +} + +# Function: encode single file with different CRF setting based on resolution + +file_encode () { + if [ ! -d "$FILEDIR/output" ]; then + mkdir "$FILEDIR/output" + fi + + FILENAME=$(basename "$INPUT_SOURCE") + RES=$(ffprobe -v error -select_streams v:0 -show_entries stream=width -of default=noprint_wrappers=1:nokey=1 "$INPUT_SOURCE") + if [[ $RES -gt 1920 ]]; then + echo "File is 4K or higher, encoding using CRF $QUALITY_4K" + ffmpeg -i "$INPUT_SOURCE" -c:v libx264 -preset slow -tune film -crf "$QUALITY_4K" -c:a copy "$FILEDIR"/output/"$FILENAME" || echo "ERROR Line $LINENO: File not encoded, unknown error occurred." 1>&2 + elif [[ $RES -le 1920 ]] && [[ -n $RES ]]; then + echo "File is HD or lower, encoding using CRF $QUALITY_HD" + ffmpeg -i "$INPUT_SOURCE" -c:v libx264 -preset slow -tune film -crf "$QUALITY_HD" -c:a copy "$FILEDIR"/output/"$FILENAME" || echo "ERROR Line $LINENO: File not encoded, unknown error occurred." 1>&2 + else + echo "ERROR Line $LINENO: Source file $INPUT_SOURCE is not a valid video file" 1>&2 + fi +} + +### SCRIPT ### + +# Check if source input is provided if [ -z "$1" ]; then printf "ERROR: You must specify a source directory\n\n" 1>&2 Help exit 1 fi -# Create output folder within source directory -if [ ! -d "$DIRECTORY/output" ]; then - mkdir "$DIRECTORY/output" +# Run function based on file or folder input +if [ -f "$1" ]; then + file_encode || error_exit "$LINENO: An error has occurred." 1>&2 +elif [ -d "$1" ]; then + folder_encode || error_exit "$LINENO: An error has occurred." 1>&2 +else + error_exit "$LINENO: Not a valid source" 1>&2 fi -# Encode each file in the directory with different CRF setting based on resolution -for FILE in "$DIRECTORY"/*.*; do - RES=$(ffprobe -v error -select_streams v:0 -show_entries stream=width -of default=noprint_wrappers=1:nokey=1 "$FILE") - FILENAME=$(basename "$FILE") - if [[ $RES -gt 1920 ]]; then - ffmpeg -i "$FILE" -c:v libx264 -preset slow -tune film -crf "$QUALITY_4K" -c:a copy "$DIRECTORY"/output/"$FILENAME" - elif [[ $RES -le 1920 ]]; then - ffmpeg -i "$FILE" -c:v libx264 -preset slow -tune film -crf "$QUALITY_HD" -c:a copy "$DIRECTORY"/output/"$FILENAME" - else - echo "$FILENAME is not a valid filetype" - fi -done +echo "File(s) encoded successfully!" + +exit 0 -exit 0 \ No newline at end of file From 331ef142e2e8c6e0fc011b83a799676c9237424e Mon Sep 17 00:00:00 2001 From: Ray Lyon Date: Fri, 4 Dec 2020 17:33:55 -0500 Subject: [PATCH 02/24] max bitrate options --- ffmpeg/ffmpeg-batch-encode.sh | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/ffmpeg/ffmpeg-batch-encode.sh b/ffmpeg/ffmpeg-batch-encode.sh index 14e3cc5..81c0307 100755 --- a/ffmpeg/ffmpeg-batch-encode.sh +++ b/ffmpeg/ffmpeg-batch-encode.sh @@ -45,10 +45,10 @@ folder_encode () { FILENAME=$(basename "$FILE") if [[ $RES -gt 1920 ]]; then echo "File is 4K or higher, encoding using CRF $QUALITY_4K" - ffmpeg -i "$FILE" -c:v libx264 -preset slow -tune film -crf "$QUALITY_4K" -c:a copy "$INPUT_SOURCE"/output/"$FILENAME" || echo "ERROR Line $LINENO: File not encoded, unknown error occurred." 1>&2 + ffmpeg -i "$FILE" -c:v libx264 -preset slow -tune film -crf "$QUALITY_4K" -maxrate 25M -bufsize 25M -c:a copy "$INPUT_SOURCE"/output/"$FILENAME" || echo "ERROR Line $LINENO: File not encoded, unknown error occurred." 1>&2 elif [[ $RES -le 1920 ]] && [[ -n $RES ]]; then echo "File is HD or lower, encoding using CRF $QUALITY_HD" - ffmpeg -i "$FILE" -c:v libx264 -preset slow -tune film -crf "$QUALITY_HD" -c:a copy "$INPUT_SOURCE"/output/"$FILENAME" || echo "ERROR Line $LINENO: File not encoded, unknown error occurred." 1>&2 + ffmpeg -i "$FILE" -c:v libx264 -preset slow -tune film -crf "$QUALITY_HD" 15M -bufsize 15M -c:a copy "$INPUT_SOURCE"/output/"$FILENAME" || echo "ERROR Line $LINENO: File not encoded, unknown error occurred." 1>&2 else echo "ERROR Line $LINENO: Source file $FILE is not a valid video file" 1>&2 echo "Skipping..." @@ -67,10 +67,10 @@ file_encode () { RES=$(ffprobe -v error -select_streams v:0 -show_entries stream=width -of default=noprint_wrappers=1:nokey=1 "$INPUT_SOURCE") if [[ $RES -gt 1920 ]]; then echo "File is 4K or higher, encoding using CRF $QUALITY_4K" - ffmpeg -i "$INPUT_SOURCE" -c:v libx264 -preset slow -tune film -crf "$QUALITY_4K" -c:a copy "$FILEDIR"/output/"$FILENAME" || echo "ERROR Line $LINENO: File not encoded, unknown error occurred." 1>&2 + ffmpeg -i "$INPUT_SOURCE" -c:v libx264 -preset slow -tune film -crf "$QUALITY_4K" -maxrate 25M -bufsize 25M -c:a copy "$FILEDIR"/output/"$FILENAME" || echo "ERROR Line $LINENO: File not encoded, unknown error occurred." 1>&2 elif [[ $RES -le 1920 ]] && [[ -n $RES ]]; then echo "File is HD or lower, encoding using CRF $QUALITY_HD" - ffmpeg -i "$INPUT_SOURCE" -c:v libx264 -preset slow -tune film -crf "$QUALITY_HD" -c:a copy "$FILEDIR"/output/"$FILENAME" || echo "ERROR Line $LINENO: File not encoded, unknown error occurred." 1>&2 + ffmpeg -i "$INPUT_SOURCE" -c:v libx264 -preset slow -tune film -crf "$QUALITY_HD" -maxrate 15M -bufsize 15M -c:a copy "$FILEDIR"/output/"$FILENAME" || echo "ERROR Line $LINENO: File not encoded, unknown error occurred." 1>&2 else echo "ERROR Line $LINENO: Source file $INPUT_SOURCE is not a valid video file" 1>&2 fi From 82bcf4079a78c8d3762f3ef398cbf6662cfee3ff Mon Sep 17 00:00:00 2001 From: Ray Lyon Date: Fri, 4 Dec 2020 18:14:40 -0500 Subject: [PATCH 03/24] fix typo --- ffmpeg/ffmpeg-batch-encode.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ffmpeg/ffmpeg-batch-encode.sh b/ffmpeg/ffmpeg-batch-encode.sh index 81c0307..d42e72f 100755 --- a/ffmpeg/ffmpeg-batch-encode.sh +++ b/ffmpeg/ffmpeg-batch-encode.sh @@ -48,7 +48,7 @@ folder_encode () { ffmpeg -i "$FILE" -c:v libx264 -preset slow -tune film -crf "$QUALITY_4K" -maxrate 25M -bufsize 25M -c:a copy "$INPUT_SOURCE"/output/"$FILENAME" || echo "ERROR Line $LINENO: File not encoded, unknown error occurred." 1>&2 elif [[ $RES -le 1920 ]] && [[ -n $RES ]]; then echo "File is HD or lower, encoding using CRF $QUALITY_HD" - ffmpeg -i "$FILE" -c:v libx264 -preset slow -tune film -crf "$QUALITY_HD" 15M -bufsize 15M -c:a copy "$INPUT_SOURCE"/output/"$FILENAME" || echo "ERROR Line $LINENO: File not encoded, unknown error occurred." 1>&2 + ffmpeg -i "$FILE" -c:v libx264 -preset slow -tune film -crf "$QUALITY_HD" -maxrate 15M -bufsize 15M -c:a copy "$INPUT_SOURCE"/output/"$FILENAME" || echo "ERROR Line $LINENO: File not encoded, unknown error occurred." 1>&2 else echo "ERROR Line $LINENO: Source file $FILE is not a valid video file" 1>&2 echo "Skipping..." From 11a2f3ce40e369f70056dbdcff53301ef24b39ce Mon Sep 17 00:00:00 2001 From: Ray Lyon <36998292+skoobasteeve@users.noreply.github.com> Date: Fri, 23 Apr 2021 17:37:59 -0400 Subject: [PATCH 04/24] Create playbook-snmp.yml --- ansible/playbook-snmp.yml | 127 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 127 insertions(+) create mode 100644 ansible/playbook-snmp.yml diff --git a/ansible/playbook-snmp.yml b/ansible/playbook-snmp.yml new file mode 100644 index 0000000..05f0332 --- /dev/null +++ b/ansible/playbook-snmp.yml @@ -0,0 +1,127 @@ +--- +# Expects snmp.conf in same directory +- name: configure snmp + hosts: active + remote_user: + become: yes + vars: + pihole01_key: "" + pihole02_key: "" + + tasks: + - name: get service facts + service_facts: + - name: get package facts + ansible.builtin.package_facts: + manager: auto + - name: check for pihole + ansible.builtin.stat: + path: "/usr/local/bin/pihole" + register: pihole + - name: install latest snmpd - debian + package: name=snmpd state=latest + when: ansible_os_family == "Debian" + - name: install latest snmpd - centos + package: name=net-snmp state=latest + when: ansible_os_family == "RedHat" + - name: install latest jq + package: name=jq state=latest + - name: copy snmpd config x86 + copy: + src: snmpd.conf + dest: "/etc/snmp/snmpd.conf" + when: ansible_architecture == "x86_64" + - name: copy snmpd config arm + copy: + src: snmpd_arm.conf + dest: "/etc/snmp/snmpd.conf" + when: ansible_architecture == "armv6l" + - name: fix extend serial permissions + ansible.builtin.file: + path: "/sys/devices/virtual/dmi/id/product_serial" + mode: '444' + when: ansible_architecture == "x86_64" + - name: cron job for extend serial permissions + ansible.builtin.lineinfile: + path: /etc/crontab + line: "@reboot chmod 444 /sys/devices/virtual/dmi/id/product_serial" + when: ansible_architecture == "x86_64" + - name: download script for extend distro + ansible.builtin.get_url: + url: "https://raw.githubusercontent.com/librenms/librenms-agent/master/snmp/distro" + dest: "/usr/bin/distro" + mode: '755' + - name: download script for extend osupdates + ansible.builtin.get_url: + url: "https://raw.githubusercontent.com/librenms/librenms-agent/master/snmp/osupdate" + dest: "/etc/snmp/osupdate" + mode: '755' + - name: download script for extend zfs + ansible.builtin.get_url: + url: "https://github.com/librenms/librenms-agent/raw/master/snmp/zfs-linux" + dest: "/etc/snmp/zfs-linux" + mode: '755' + when: "'zfs-zed' in ansible_facts.packages" + - name: download script for extend docker + ansible.builtin.get_url: + url: "https://github.com/librenms/librenms-agent/raw/master/snmp/docker-stats.sh" + dest: "/etc/snmp/docker-stats.sh" + mode: '755' + when: "'docker' in services" + - name: download script for extend pihole + ansible.builtin.get_url: + url: "https://github.com/librenms/librenms-agent/raw/master/snmp/pi-hole" + dest: "/etc/snmp/pi-hole" + mode: '755' + when: pihole.stat.exists + - name: add api key to pihole script for pihole01 + replace: + path: "/etc/snmp/pi-hole" + regexp: 'API_AUTH_KEY=""' + replace: 'API_AUTH_KEY="{{ pihole01_key }}"' + when: ansible_hostname == "pihole01" + - name: add api key to pihole script for pihole02 + replace: + path: "/etc/snmp/pi-hole" + regexp: 'API_AUTH_KEY=""' + replace: 'API_AUTH_KEY="{{ pihole02_key }}"' + when: ansible_hostname == "pihole02" + - name: enable extend nfs-server + ansible.builtin.lineinfile: + path: "/etc/snmp/snmpd.conf" + line: "extend nfs-server /bin/cat /proc/net/rpc/nfsd" + when: "'nfs-kernel-server' in ansible_facts.services" + - name: enable extend zfs + ansible.builtin.lineinfile: + path: "/etc/snmp/snmpd.conf" + line: "extend zfs '/usr/bin/sudo /etc/snmp/zfs-linux'" + when: "'zfs-zed' in ansible_facts.packages" + - name: update sudoers file for extend zfs + ansible.builtin.lineinfile: + path: "/etc/sudoers" + line: "Debian-snmp ALL=(ALL) NOPASSWD: /etc/snmp/zfs-linux" + when: "'zfs-zed' in ansible_facts.packages" + - name: enable extend docker + when: "'docker' in services" + ansible.builtin.lineinfile: + path: "/etc/snmp/snmpd.conf" + line: "extend docker /usr/bin/sudo /etc/snmp/docker-stats.sh" + - name: enable extend pihole + when: pihole.stat.exists + ansible.builtin.lineinfile: + path: "/etc/snmp/snmpd.conf" + line: "extend pi-hole /etc/snmp/pi-hole" + - name: update sudoers file for extend docker + when: "'docker' in services" + ansible.builtin.lineinfile: + path: "/etc/sudoers" + line: "Debian-snmp ALL=(ALL) NOPASSWD: /etc/snmp/docker-stats.sh" + - name: enable extend osupdates + ansible.builtin.lineinfile: + path: "/etc/snmp/snmpd.conf" + line: "extend osupdate /etc/snmp/osupdate" + - name: enable and restart snmpd.service + ansible.builtin.systemd: + state: restarted + enabled: yes + name: snmpd From 50cfd3a11515777267b91e9fddc5799e77da38d2 Mon Sep 17 00:00:00 2001 From: Ray Lyon <36998292+skoobasteeve@users.noreply.github.com> Date: Fri, 23 Apr 2021 17:39:09 -0400 Subject: [PATCH 05/24] Create snmpd.conf --- ansible/snmpd.conf | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 ansible/snmpd.conf diff --git a/ansible/snmpd.conf b/ansible/snmpd.conf new file mode 100644 index 0000000..51fde74 --- /dev/null +++ b/ansible/snmpd.conf @@ -0,0 +1,27 @@ +# Change RANDOMSTRINGGOESHERE to your preferred SNMP community string +com2sec readonly default RANDOMSTRINGGOESHERE + +group MyROGroup v2c readonly +view all included .1 80 +access MyROGroup "" any noauth exact all none none + +syslocation Home +syscontact Ray Lyon + +agentAddress udp:161,udp6:[::1]:161 +rocommunity RANDOMSTRINGGOESHERE + +#OS Distribution Detection +extend distro /usr/bin/distro + +#Hardware Detection +# (uncomment for x86 platforms) +extend manufacturer '/bin/cat /sys/devices/virtual/dmi/id/sys_vendor' +extend hardware '/bin/cat /sys/devices/virtual/dmi/id/product_name' +extend serial '/bin/cat /sys/devices/virtual/dmi/id/product_serial' + +# (uncomment for ARM platforms) +#extend hardware '/bin/cat /sys/firmware/devicetree/base/model' +#extend serial '/bin/cat /sys/firmware/devicetree/base/serial-number +# +# From fb99dafe6784ee2d8cca96796df433b442489a5d Mon Sep 17 00:00:00 2001 From: Ray Lyon <36998292+skoobasteeve@users.noreply.github.com> Date: Fri, 23 Apr 2021 17:39:27 -0400 Subject: [PATCH 06/24] fixed spelling --- ansible/playbook-snmp.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/playbook-snmp.yml b/ansible/playbook-snmp.yml index 05f0332..f3c73f8 100644 --- a/ansible/playbook-snmp.yml +++ b/ansible/playbook-snmp.yml @@ -1,5 +1,5 @@ --- -# Expects snmp.conf in same directory +# Expects snmpd.conf in same directory - name: configure snmp hosts: active remote_user: From 082222119007c6a8f12581f2b99f371b2dea516b Mon Sep 17 00:00:00 2001 From: Ray Lyon <36998292+skoobasteeve@users.noreply.github.com> Date: Sat, 24 Apr 2021 19:33:25 -0400 Subject: [PATCH 07/24] fixed snmpd logging verbosity --- ansible/playbook-snmp.yml | 39 +++++++++++++++++++++++++++++++-------- 1 file changed, 31 insertions(+), 8 deletions(-) diff --git a/ansible/playbook-snmp.yml b/ansible/playbook-snmp.yml index f3c73f8..058ebe8 100644 --- a/ansible/playbook-snmp.yml +++ b/ansible/playbook-snmp.yml @@ -1,8 +1,8 @@ --- -# Expects snmpd.conf in same directory +# Expects snmpd.conf and snmpd_arm.conf in same directory - name: configure snmp hosts: active - remote_user: + remote_user: raylyon become: yes vars: pihole01_key: "" @@ -75,16 +75,18 @@ mode: '755' when: pihole.stat.exists - name: add api key to pihole script for pihole01 - replace: + ansible.builtin.lineinfile: path: "/etc/snmp/pi-hole" - regexp: 'API_AUTH_KEY=""' - replace: 'API_AUTH_KEY="{{ pihole01_key }}"' + regexp: '^API_AUTH_KEY=' + line: 'API_AUTH_KEY="{{ pihole01_key }}"' + backrefs: yes when: ansible_hostname == "pihole01" - name: add api key to pihole script for pihole02 - replace: + ansible.builtin.lineinfile: path: "/etc/snmp/pi-hole" - regexp: 'API_AUTH_KEY=""' - replace: 'API_AUTH_KEY="{{ pihole02_key }}"' + regexp: '^API_AUTH_KEY=' + line: 'API_AUTH_KEY="{{ pihole02_key }}"' + backrefs: yes when: ansible_hostname == "pihole02" - name: enable extend nfs-server ansible.builtin.lineinfile: @@ -120,8 +122,29 @@ ansible.builtin.lineinfile: path: "/etc/snmp/snmpd.conf" line: "extend osupdate /etc/snmp/osupdate" + - name: set ExecStart options in service file - ubuntu + ansible.builtin.lineinfile: + path: "/lib/systemd/system/snmpd.service" + regexp: '^ExecStart=' + line: "ExecStart=/usr/sbin/snmpd -LS4d -Lf /dev/null -u Debian-snmp -g Debian-snmp -I -smux,mteTrigger,mteTriggerConf -f" + backrefs: yes + when: ansible_os_family == "Debian" + - name: reload systemd configs - ubuntu + ansible.builtin.systemd: + daemon_reload: yes + when: ansible_os_family == "Debian" + - name: set snmpdopts - centos + ansible.builtin.lineinfile: + path: "/etc/sysconfig/snmpd" + regexp: '^# OPTIONS=|^OPTIONS=' + line: 'OPTIONS="-LS4-6d"' + when: ansible_os_family == "RedHat" - name: enable and restart snmpd.service ansible.builtin.systemd: state: restarted enabled: yes name: snmpd + - name: verify the snmpd service is running + ansible.builtin.systemd: + state: started + name: snmpd From 34fa70a9d8a6922461b65c905510467ad235a3e3 Mon Sep 17 00:00:00 2001 From: Ray Lyon <36998292+skoobasteeve@users.noreply.github.com> Date: Fri, 10 Sep 2021 16:34:43 -0400 Subject: [PATCH 08/24] Create netlify-form-downloader.py --- netlify/netlify-form-downloader.py | 108 +++++++++++++++++++++++++++++ 1 file changed, 108 insertions(+) create mode 100644 netlify/netlify-form-downloader.py diff --git a/netlify/netlify-form-downloader.py b/netlify/netlify-form-downloader.py new file mode 100644 index 0000000..fada12f --- /dev/null +++ b/netlify/netlify-form-downloader.py @@ -0,0 +1,108 @@ +#!/usr/bin/python3 + +""" + +This script gets uploaded files from a Netlify Forms submission, renames them, and uploads them to a Nextcloud folder. +I originally used it to download vaccine cards that my wedding guests submitted and move them to a shared folder. + +Required Packages: +pip install requests +pip install webdavclient3 + + +""" + +import requests +from webdav3.client import Client +import urllib.request +import os +import shutil + +### USER VARIABLES ### + +# Netlify +USERNAME="" +OAUTH_TOKEN="" +SITE_ID="" +FORM_ID="" + +# Nextcloud +NEXTCLOUD_DIR = "" +NEXTCLOUD_USER = "" +NEXTCLOUD_PASS = "" +NEXTCLOUD_URL = "" + +#### DON'T EDIT BELOW THIS LINE #### + +headers = {'Authorization': 'Bearer ' + OAUTH_TOKEN , 'User-Agent': 'MyApp (' + USERNAME + ')'} +form_submissions = requests.get(f"https://api.netlify.com/api/v1/sites/{SITE_ID}/forms/{FORM_ID}/submissions", headers=headers).json() +vaccine_cards = {} +webdav_options = { + 'webdav_hostname': NEXTCLOUD_URL, + 'webdav_login': NEXTCLOUD_USER, + 'webdav_password': NEXTCLOUD_PASS +} +client = Client(webdav_options) +existing_cards = client.list(NEXTCLOUD_DIR) +new_cards = [] +all_cards = [] + +#### FUNCTIONS #### + +def build_dict(): + for entry in form_submissions: + name = entry["data"]["name"] + card_img = entry["data"]["vaccine_card"]["url"] + vaccine_cards[name] = card_img + +def download_cards(): + print("Downloading cards from Netlify...") + for name, card in vaccine_cards.items(): + response = urllib.request.urlopen(card) + info = response.info() + extension = "." + str(info.get_content_subtype()) + name_clean = name.strip() + output_file = name_clean.replace(' ', '_') + extension + all_cards.append(output_file) + if output_file not in existing_cards: + new_cards.append(output_file) + file_download = requests.get(card, stream=True) + if os.path.exists('tmp/') == False: + os.makedirs('tmp/') + print(output_file) + with open(f'tmp/{output_file}', 'wb') as f: + for chunk in file_download.iter_content(2000): + f.write(chunk) + else: + continue + +def upload_cards(): + num_cards = len(new_cards) + current_card = 0 + print("") + print("Uploading cards to Nextcloud...") + for card in os.listdir("tmp"): + if card in new_cards: + current_card += 1 + print(f"Uploading card {current_card} of {num_cards}") + client.upload_sync(remote_path=f'{NEXTCLOUD_DIR}/{card}', local_path=f"tmp/{card}") + else: + continue + print("Done!") + +def main(): + build_dict() + if len(vaccine_cards) > (len(existing_cards) - 1): + download_cards() + else: + print("Nothing new to download!") + if new_cards: + upload_cards() + if os.path.exists('tmp/') == True: + print("") + print("Cleaning up...") + shutil.rmtree('tmp/') + + +if __name__ == '__main__': + main() From 5788dd5a5f6d4238eced799adc9d16e4060d72a0 Mon Sep 17 00:00:00 2001 From: Ray Lyon <36998292+skoobasteeve@users.noreply.github.com> Date: Fri, 10 Sep 2021 16:47:06 -0400 Subject: [PATCH 09/24] Added comments --- netlify/netlify-form-downloader.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/netlify/netlify-form-downloader.py b/netlify/netlify-form-downloader.py index fada12f..9e01668 100644 --- a/netlify/netlify-form-downloader.py +++ b/netlify/netlify-form-downloader.py @@ -34,6 +34,7 @@ NEXTCLOUD_URL = "" #### DON'T EDIT BELOW THIS LINE #### +# Netlify API calls headers = {'Authorization': 'Bearer ' + OAUTH_TOKEN , 'User-Agent': 'MyApp (' + USERNAME + ')'} form_submissions = requests.get(f"https://api.netlify.com/api/v1/sites/{SITE_ID}/forms/{FORM_ID}/submissions", headers=headers).json() vaccine_cards = {} @@ -49,12 +50,14 @@ all_cards = [] #### FUNCTIONS #### +# Creates a dictionary from the Netlify form data { "Name": "" } def build_dict(): for entry in form_submissions: name = entry["data"]["name"] card_img = entry["data"]["vaccine_card"]["url"] vaccine_cards[name] = card_img +# Downloads files from Netlify based on their URL and renames the files as "First_Last.jpg(png, pdf, etc)" def download_cards(): print("Downloading cards from Netlify...") for name, card in vaccine_cards.items(): @@ -75,7 +78,8 @@ def download_cards(): f.write(chunk) else: continue - + +# Uploads files to the specified Nextcloud/WebDAV folder if they don't already exist def upload_cards(): num_cards = len(new_cards) current_card = 0 From 7d8d312d6d25e992e61e7b3b3154d1a5f4639f6a Mon Sep 17 00:00:00 2001 From: Ray Lyon <36998292+skoobasteeve@users.noreply.github.com> Date: Fri, 10 Sep 2021 16:48:20 -0400 Subject: [PATCH 10/24] Create foxpass_radius_logs.py --- foxpass/foxpass_radius_logs.py | 150 +++++++++++++++++++++++++++++++++ 1 file changed, 150 insertions(+) create mode 100644 foxpass/foxpass_radius_logs.py diff --git a/foxpass/foxpass_radius_logs.py b/foxpass/foxpass_radius_logs.py new file mode 100644 index 0000000..988bca5 --- /dev/null +++ b/foxpass/foxpass_radius_logs.py @@ -0,0 +1,150 @@ +#!/usr/local/bin/python3 + +""" +This script pulls RADIUS logs from your Foxpass instance and allows you to parse them by date, user, IP, or connection outcome. +Logs can be printed (in pretty colors) or exported in CSV format. +Required packages: +pip install requests +To run: +python foxpass_radius_logs.py +By default the script will print color-coded RADIUS logs from the last (5) days. You can use the optional arguments below: +--hours - How far back to show the logs in Hours. +--user - Filter by user. +--location - Filter by RADIUS Client, based on the items defined in the OFFICE_IPS dict. +--outcome - Filter by outcome of the connection, specify True or False. +--csv - Output the logs to a CSV file, specify the filename and path. +""" + +from datetime import datetime, timedelta, timezone +import requests +import argparse +import csv + +##### EDIT THESE ##### +FOXPASS_API_TOKEN = "" +# RADIUS clients, can be called with the --location argument. +OFFICE_IPS = { + "office1":"", + "office2":"", + "office3":"", + } + +# "YYYY-MM-DDTHH:MMZ". STARTDATE Default 5 days ago, ENDDATE is current day/time in UTC. Can be changed with the --hours argument. +STARTDATE = (datetime.now(timezone.utc) - timedelta(days=5)).strftime('%Y-%m-%dT%H:%MZ') +ENDDATE = datetime.now(timezone.utc).strftime('%Y-%m-%dT%H:%MZ') + +FOXPASS_URL = "https://api.foxpass.com/v1/logs/radius/" +HEADERS = {'Authorization': 'Token ' + FOXPASS_API_TOKEN} +PAGEREQUEST = requests.post(FOXPASS_URL, json={"from": STARTDATE, "to": ENDDATE}, headers=HEADERS).json() +PAGES = PAGEREQUEST["numPages"] + +class bcolors: + HEADER = '\033[95m' + OKBLUE = '\033[94m' + OKCYAN = '\033[96m' + OKGREEN = '\033[92m' + WARNING = '\033[93m' + FAIL = '\033[91m' + ENDC = '\033[0m' + BOLD = '\033[1m' + UNDERLINE = '\033[4m' + +def get_args(): + parser = argparse.ArgumentParser(description='Pull and parse RADIUS logs from your Foxpass environment') + parser.add_argument('--user', help='Filter logs by username') + parser.add_argument('--outcome', help='Filter logs by connection outcome: True or False') + parser.add_argument('--hours', type=int, help='How far back to check the logs in hours') + parser.add_argument('--location', help='Filter logs by location: tlv, tlv-backup, nyc, nyc-backup, sha') + parser.add_argument('--csv', help='Export a CSV of the log data to the specified filename and path') + return parser.parse_args() + +# Builds an if statement to filter the logs based on user arguments +def build_query(username=None, outcome=None, location=None): + query_string = "" + if username != None: + query_string += f"log['username']=='{username}' and " + if location != None: + query_string += f"log['ipAddress']=='{location}' and " + if outcome != None: + query_string += f"log['success']=={outcome}" + if query_string != "": + # If the string ends with "and", remove it before returning. + if query_string[-2] == 'd': + query_string = query_string[:-5] + return query_string + +# Pulls logs from Foxpass and stores them +def get_logs(): + p = 0 + logs_full = [] + while p < PAGES: + p += 1 + request = requests.post(FOXPASS_URL, json={"from": STARTDATE, "to": ENDDATE, "page": p, "ascending": True}, headers=HEADERS).json() + request_clean = request["data"] + logs_full.append(request_clean) + return logs_full + +# Prints or exports all logs for the specified time period +def lookup_all(logs, csv_arg=None, csv_writer=None): + p=0 + while p < PAGES: + for log in logs[p]: + if csv_arg == None: + print_logs(log) + elif csv_arg != None: + csv_export(log, csv_writer) + p += 1 + +# Prints or exports logs based on user-provided filter arguments for the specified time period +def lookup_filter(logs, if_statement, csvarg=None, csv_writer=None): + p=0 + while p < PAGES: + for log in logs[p]: + if eval(if_statement) and csvarg == None: + print_logs(log) + elif eval(if_statement) and csvarg != None: + csv_export(log, csv_writer) + p += 1 + +def csv_export(log, csv_writer): + csv_writer.writerow([log["timestamp"], log["username"], log["ipAddress"], log["message"], log["success"]]) + +# Determines start time based on the --hours argument +def start_time(hours): + d = datetime.now(timezone.utc) - timedelta(hours=hours) + return d.strftime('%Y-%m-%dT%H:%MZ') + +def print_logs(sourcedict): + print(bcolors.OKCYAN + sourcedict["timestamp"],bcolors.OKGREEN + sourcedict["username"],bcolors.WARNING + sourcedict["ipAddress"],bcolors.FAIL + sourcedict["message"],bcolors.OKBLUE + "Success:",sourcedict["success"]) + +def main(): + global STARTDATE + args = get_args() + + if args.csv != None: + csv_open = open(args.csv, 'w', newline='') + csv_writer = csv.writer(csv_open) + csv_writer.writerow(["TIMESTAMP (UTC)","USERNAME","IP ADDRESS","MESSAGE","SUCCESS"]) + else: + csv_writer = None + + if args.hours: + STARTDATE = start_time(args.hours) + + if args.location != None: + location_ip = OFFICE_IPS[args.location] + else: + location_ip = None + + if_statement = build_query(args.user, args.outcome, location_ip) + + logs = get_logs() + + if if_statement == "": + lookup_all(logs, args.csv, csv_writer) + else: + lookup_filter(logs, if_statement, args.csv, csv_writer) + + +if __name__ == '__main__': + main() From 9df2412231aa9d19ff60484995a2ce7f99edd883 Mon Sep 17 00:00:00 2001 From: Ray Lyon <36998292+skoobasteeve@users.noreply.github.com> Date: Tue, 14 Sep 2021 18:12:58 -0400 Subject: [PATCH 11/24] better comparison for new files --- netlify/netlify-form-downloader.py | 33 +++++++++++++++++++++++------- 1 file changed, 26 insertions(+), 7 deletions(-) diff --git a/netlify/netlify-form-downloader.py b/netlify/netlify-form-downloader.py index 9e01668..00baa09 100644 --- a/netlify/netlify-form-downloader.py +++ b/netlify/netlify-form-downloader.py @@ -44,25 +44,44 @@ webdav_options = { 'webdav_password': NEXTCLOUD_PASS } client = Client(webdav_options) -existing_cards = client.list(NEXTCLOUD_DIR) +nc = client.list(NEXTCLOUD_DIR) +existing_cards = nc[1:] new_cards = [] all_cards = [] #### FUNCTIONS #### -# Creates a dictionary from the Netlify form data { "Name": "" } def build_dict(): for entry in form_submissions: name = entry["data"]["name"] card_img = entry["data"]["vaccine_card"]["url"] vaccine_cards[name] = card_img -# Downloads files from Netlify based on their URL and renames the files as "First_Last.jpg(png, pdf, etc)" +def card_sizes_netlify(): + netlify_cards = {} + for name, card in vaccine_cards.items(): + response = urllib.request.urlopen(card) + info = response.headers + filesize = info['Content-Length'] + extension = "." + str(info.get_content_subtype()) + name_clean = name.strip() + output_file = name_clean.replace(' ', '_') + extension + netlify_cards[output_file] = filesize + return netlify_cards + +def card_sizes_nextcloud(): + nextcloud_cards = {} + for card in existing_cards: + card_info = client.info(NEXTCLOUD_DIR + card) + filesize = card_info['size'] + nextcloud_cards[card] = filesize + return nextcloud_cards + def download_cards(): print("Downloading cards from Netlify...") for name, card in vaccine_cards.items(): response = urllib.request.urlopen(card) - info = response.info() + info = response.headers extension = "." + str(info.get_content_subtype()) name_clean = name.strip() output_file = name_clean.replace(' ', '_') + extension @@ -78,8 +97,7 @@ def download_cards(): f.write(chunk) else: continue - -# Uploads files to the specified Nextcloud/WebDAV folder if they don't already exist + def upload_cards(): num_cards = len(new_cards) current_card = 0 @@ -96,7 +114,8 @@ def upload_cards(): def main(): build_dict() - if len(vaccine_cards) > (len(existing_cards) - 1): + print("Checking for new vaccine cards...") + if card_sizes_netlify() != card_sizes_nextcloud(): download_cards() else: print("Nothing new to download!") From 23bf4392e4d0fc8493ca557c991fe5e3f3f1c4d1 Mon Sep 17 00:00:00 2001 From: Ray Lyon Date: Sat, 19 Feb 2022 18:55:37 -0500 Subject: [PATCH 12/24] initial working script --- streaming-check/movie_check.py | 72 ++++++++++++++++++++++++++++++++++ 1 file changed, 72 insertions(+) create mode 100644 streaming-check/movie_check.py diff --git a/streaming-check/movie_check.py b/streaming-check/movie_check.py new file mode 100644 index 0000000..7261cd1 --- /dev/null +++ b/streaming-check/movie_check.py @@ -0,0 +1,72 @@ +#!/usr/bin/python3 + +import requests +import urllib +from datetime import datetime +import os + +tmdb_api_token = os.environ.get("TMDB_API_TOKEN") +sa_api_token = os.environ.get("SA_API_TOKEN") + +tmdb_url = "https://api.themoviedb.org/3" +tmdb_headers = { + 'Authorization': f'Bearer {tmdb_api_token}', + 'Content-Type': 'application/json;charset=utf-8', + 'Accept': 'application/json;charset=utf-8' +} + +sa_url = "https://streaming-availability.p.rapidapi.com/get/basic" +sa_headers = { + 'x-rapidapi-host': "streaming-availability.p.rapidapi.com", + 'x-rapidapi-key': sa_api_token + } + +movie = "12 angry men" +movie_safe = urllib.parse.quote_plus(movie) + +tmdb_search = requests.get(f"{tmdb_url}/search/movie?language=en-US&query={movie_safe}&page=1&include_adult=false", headers=tmdb_headers).json() +movie_id = tmdb_search['results'][0]['id'] +movie_tile = tmdb_search['results'][0]['title'] +movie_release = tmdb_search['results'][0]['release_date'] + +sa_querystring = {"country":"us","tmdb_id":f"movie/{movie_id}","output_language":"en"} + + + +sa_response = requests.request("GET", sa_url, headers=sa_headers, params=sa_querystring).json() + +services_list = [] +services = sa_response["streamingInfo"] + +print(movie_tile + f" ({movie_release})") +for s in services: + countries = sa_response["streamingInfo"][s] + for c in countries: + leaving_epoch = sa_response["streamingInfo"][s][c]["leaving"] + leaving_date = datetime.fromtimestamp(int(leaving_epoch)).strftime('%Y-%m-%d') + link = sa_response["streamingInfo"][s][c]["link"] + print(f"Available on {s}") + if leaving_epoch != 0: + print(f"Will be leaving {s} on {leaving_date}") + print(f"Watch here: {link}") +# country = response["streamingInfo"][services] +# leaving = response["streamingInfo"][services]["leaving"] +# link = response["streamingInfo"][services]["link"] + +#print(services) + + +# changes = {} +# for x in services: +# changes[x] = response["streamingInfo"][x][x]['leaving'] +# print(changes) + + +# for s in services: +# services_list.append(s) + +# print(services_list) + +# for s in services['hulu']: +# print(s) +#print(json.dumps(response, indent=4, sort_keys=True)) \ No newline at end of file From a725bc4886f3bf6b34f46390d9d8193938b6acfe Mon Sep 17 00:00:00 2001 From: Ray Lyon Date: Sat, 19 Feb 2022 19:11:00 -0500 Subject: [PATCH 13/24] switched date to year --- streaming-check/movie_check.py | 31 +++---------------------------- 1 file changed, 3 insertions(+), 28 deletions(-) diff --git a/streaming-check/movie_check.py b/streaming-check/movie_check.py index 7261cd1..f863ad1 100644 --- a/streaming-check/movie_check.py +++ b/streaming-check/movie_check.py @@ -27,18 +27,14 @@ movie_safe = urllib.parse.quote_plus(movie) tmdb_search = requests.get(f"{tmdb_url}/search/movie?language=en-US&query={movie_safe}&page=1&include_adult=false", headers=tmdb_headers).json() movie_id = tmdb_search['results'][0]['id'] movie_tile = tmdb_search['results'][0]['title'] -movie_release = tmdb_search['results'][0]['release_date'] +movie_release = datetime.strptime(tmdb_search['results'][0]['release_date'], "%Y-%m-%d") sa_querystring = {"country":"us","tmdb_id":f"movie/{movie_id}","output_language":"en"} - - - sa_response = requests.request("GET", sa_url, headers=sa_headers, params=sa_querystring).json() -services_list = [] services = sa_response["streamingInfo"] -print(movie_tile + f" ({movie_release})") +print(movie_tile + f" ({movie_release.year})") for s in services: countries = sa_response["streamingInfo"][s] for c in countries: @@ -48,25 +44,4 @@ for s in services: print(f"Available on {s}") if leaving_epoch != 0: print(f"Will be leaving {s} on {leaving_date}") - print(f"Watch here: {link}") -# country = response["streamingInfo"][services] -# leaving = response["streamingInfo"][services]["leaving"] -# link = response["streamingInfo"][services]["link"] - -#print(services) - - -# changes = {} -# for x in services: -# changes[x] = response["streamingInfo"][x][x]['leaving'] -# print(changes) - - -# for s in services: -# services_list.append(s) - -# print(services_list) - -# for s in services['hulu']: -# print(s) -#print(json.dumps(response, indent=4, sort_keys=True)) \ No newline at end of file + print(f"Watch here: {link}") \ No newline at end of file From 7e4c86b921f5fdef3bd49533b9e85a558f72c310 Mon Sep 17 00:00:00 2001 From: Ray Lyon Date: Sat, 19 Feb 2022 20:15:49 -0500 Subject: [PATCH 14/24] Error handling and spelling correction --- streaming-check/movie_check.py | 29 +++++++++++++++++++++++++++-- 1 file changed, 27 insertions(+), 2 deletions(-) diff --git a/streaming-check/movie_check.py b/streaming-check/movie_check.py index f863ad1..1e2a861 100644 --- a/streaming-check/movie_check.py +++ b/streaming-check/movie_check.py @@ -4,6 +4,7 @@ import requests import urllib from datetime import datetime import os +import sys tmdb_api_token = os.environ.get("TMDB_API_TOKEN") sa_api_token = os.environ.get("SA_API_TOKEN") @@ -21,10 +22,15 @@ sa_headers = { 'x-rapidapi-key': sa_api_token } -movie = "12 angry men" +movie = "finch" movie_safe = urllib.parse.quote_plus(movie) tmdb_search = requests.get(f"{tmdb_url}/search/movie?language=en-US&query={movie_safe}&page=1&include_adult=false", headers=tmdb_headers).json() + +if not tmdb_search["results"]: + print("I'm having trouble finding that movie. Check your spelling and try again.") + exit() + movie_id = tmdb_search['results'][0]['id'] movie_tile = tmdb_search['results'][0]['title'] movie_release = datetime.strptime(tmdb_search['results'][0]['release_date'], "%Y-%m-%d") @@ -34,14 +40,33 @@ sa_response = requests.request("GET", sa_url, headers=sa_headers, params=sa_quer services = sa_response["streamingInfo"] +def services_speller(service): + if service == "hbo": + service_proper = "HBO Max" + if service == "hulu": + service_proper = "Hulu" + if service == "prime": + service_proper = "Amazon Prime" + if service == "netflix": + service_proper = "Netflix" + if service == "disney": + service_proper = "Disney+" + if service == "apple": + service_proper = "Apple TV+" + return service_proper + + + print(movie_tile + f" ({movie_release.year})") +if not services: + print("Movie not available for streaming :(") for s in services: countries = sa_response["streamingInfo"][s] for c in countries: leaving_epoch = sa_response["streamingInfo"][s][c]["leaving"] leaving_date = datetime.fromtimestamp(int(leaving_epoch)).strftime('%Y-%m-%d') link = sa_response["streamingInfo"][s][c]["link"] - print(f"Available on {s}") + print(f"Available on {services_speller(s)}") if leaving_epoch != 0: print(f"Will be leaving {s} on {leaving_date}") print(f"Watch here: {link}") \ No newline at end of file From 35d75c4b69d240e412a7e773be385582218fca98 Mon Sep 17 00:00:00 2001 From: Ray Lyon Date: Sat, 19 Feb 2022 23:53:20 -0500 Subject: [PATCH 15/24] Rating check from TMDB --- streaming-check/movie_check.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/streaming-check/movie_check.py b/streaming-check/movie_check.py index 1e2a861..a076cce 100644 --- a/streaming-check/movie_check.py +++ b/streaming-check/movie_check.py @@ -4,7 +4,6 @@ import requests import urllib from datetime import datetime import os -import sys tmdb_api_token = os.environ.get("TMDB_API_TOKEN") sa_api_token = os.environ.get("SA_API_TOKEN") @@ -22,7 +21,7 @@ sa_headers = { 'x-rapidapi-key': sa_api_token } -movie = "finch" +movie = "eternals" movie_safe = urllib.parse.quote_plus(movie) tmdb_search = requests.get(f"{tmdb_url}/search/movie?language=en-US&query={movie_safe}&page=1&include_adult=false", headers=tmdb_headers).json() @@ -34,6 +33,8 @@ if not tmdb_search["results"]: movie_id = tmdb_search['results'][0]['id'] movie_tile = tmdb_search['results'][0]['title'] movie_release = datetime.strptime(tmdb_search['results'][0]['release_date'], "%Y-%m-%d") +movie_rating = tmdb_search['results'][0]['vote_average'] + sa_querystring = {"country":"us","tmdb_id":f"movie/{movie_id}","output_language":"en"} sa_response = requests.request("GET", sa_url, headers=sa_headers, params=sa_querystring).json() @@ -58,6 +59,7 @@ def services_speller(service): print(movie_tile + f" ({movie_release.year})") +print(f"Rating: {movie_rating}") if not services: print("Movie not available for streaming :(") for s in services: From c3cde35d328671631e0c8f6c75838c2e91aa879d Mon Sep 17 00:00:00 2001 From: Ray Lyon Date: Sun, 20 Feb 2022 00:02:39 -0500 Subject: [PATCH 16/24] user input for movie title --- streaming-check/movie_check.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/streaming-check/movie_check.py b/streaming-check/movie_check.py index a076cce..240192c 100644 --- a/streaming-check/movie_check.py +++ b/streaming-check/movie_check.py @@ -21,7 +21,7 @@ sa_headers = { 'x-rapidapi-key': sa_api_token } -movie = "eternals" +movie = input("Enter a movie: ") movie_safe = urllib.parse.quote_plus(movie) tmdb_search = requests.get(f"{tmdb_url}/search/movie?language=en-US&query={movie_safe}&page=1&include_adult=false", headers=tmdb_headers).json() @@ -61,7 +61,7 @@ def services_speller(service): print(movie_tile + f" ({movie_release.year})") print(f"Rating: {movie_rating}") if not services: - print("Movie not available for streaming :(") + print("Streaming not available :(") for s in services: countries = sa_response["streamingInfo"][s] for c in countries: From cbaec1a4e73c1dc6a684e8ac9f5fb0faddd1395a Mon Sep 17 00:00:00 2001 From: Ray Lyon Date: Sun, 20 Feb 2022 13:47:39 -0500 Subject: [PATCH 17/24] -removed unnecessary loop -error handling for unknown year -error handling for 404 -optional --year argument --- streaming-check/movie_check.py | 60 +++++++++++++++++++++++----------- 1 file changed, 41 insertions(+), 19 deletions(-) diff --git a/streaming-check/movie_check.py b/streaming-check/movie_check.py index 240192c..8cdb1a2 100644 --- a/streaming-check/movie_check.py +++ b/streaming-check/movie_check.py @@ -4,6 +4,12 @@ import requests import urllib from datetime import datetime import os +import argparse + +def get_args(): + parser = argparse.ArgumentParser(description='Search movie streaming availability.') + parser.add_argument('--year', type=int, help='Specify movie release year') + return parser.parse_args() tmdb_api_token = os.environ.get("TMDB_API_TOKEN") sa_api_token = os.environ.get("SA_API_TOKEN") @@ -21,10 +27,14 @@ sa_headers = { 'x-rapidapi-key': sa_api_token } +args = get_args() movie = input("Enter a movie: ") movie_safe = urllib.parse.quote_plus(movie) -tmdb_search = requests.get(f"{tmdb_url}/search/movie?language=en-US&query={movie_safe}&page=1&include_adult=false", headers=tmdb_headers).json() +if args.year: + tmdb_search = requests.get(f"{tmdb_url}/search/movie?language=en-US&query={movie_safe}&page=1&include_adult=false&primary_release_year={args.year}", headers=tmdb_headers).json() +else: + tmdb_search = requests.get(f"{tmdb_url}/search/movie?language=en-US&query={movie_safe}&page=1&include_adult=false", headers=tmdb_headers).json() if not tmdb_search["results"]: print("I'm having trouble finding that movie. Check your spelling and try again.") @@ -32,43 +42,55 @@ if not tmdb_search["results"]: movie_id = tmdb_search['results'][0]['id'] movie_tile = tmdb_search['results'][0]['title'] -movie_release = datetime.strptime(tmdb_search['results'][0]['release_date'], "%Y-%m-%d") +movie_release_check = tmdb_search['results'][0]['release_date'] + +if movie_release_check: + movie_release = datetime.strptime(tmdb_search['results'][0]['release_date'], "%Y-%m-%d") +else: movie_release = "???" + movie_rating = tmdb_search['results'][0]['vote_average'] - sa_querystring = {"country":"us","tmdb_id":f"movie/{movie_id}","output_language":"en"} -sa_response = requests.request("GET", sa_url, headers=sa_headers, params=sa_querystring).json() +sa_request = requests.request("GET", sa_url, headers=sa_headers, params=sa_querystring) + + +if sa_request.status_code == 404: + print("I'm having trouble finding that movie. Check your spelling and try again.") + exit() + +sa_response = sa_request.json() services = sa_response["streamingInfo"] def services_speller(service): if service == "hbo": service_proper = "HBO Max" - if service == "hulu": + elif service == "hulu": service_proper = "Hulu" - if service == "prime": + elif service == "prime": service_proper = "Amazon Prime" - if service == "netflix": + elif service == "netflix": service_proper = "Netflix" - if service == "disney": + elif service == "disney": service_proper = "Disney+" - if service == "apple": + elif service == "apple": service_proper = "Apple TV+" + elif service == "paramount": + service_proper = "Paramount+" + else: + return service return service_proper - print(movie_tile + f" ({movie_release.year})") print(f"Rating: {movie_rating}") if not services: print("Streaming not available :(") for s in services: - countries = sa_response["streamingInfo"][s] - for c in countries: - leaving_epoch = sa_response["streamingInfo"][s][c]["leaving"] - leaving_date = datetime.fromtimestamp(int(leaving_epoch)).strftime('%Y-%m-%d') - link = sa_response["streamingInfo"][s][c]["link"] - print(f"Available on {services_speller(s)}") - if leaving_epoch != 0: - print(f"Will be leaving {s} on {leaving_date}") - print(f"Watch here: {link}") \ No newline at end of file + leaving_epoch = sa_response["streamingInfo"][s]["us"]["leaving"] + leaving_date = datetime.fromtimestamp(int(leaving_epoch)).strftime('%Y-%m-%d') + link = sa_response["streamingInfo"][s]["us"]["link"] + print(f"Available on {services_speller(s)}") + if leaving_epoch != 0: + print(f"Will be leaving {s} on {leaving_date}") + print(f"Watch here: {link}") \ No newline at end of file From b732735187d58c58859b9a89a606dbb3f4d84068 Mon Sep 17 00:00:00 2001 From: Ray Lyon Date: Mon, 28 Mar 2022 18:59:11 -0400 Subject: [PATCH 18/24] functions, better formatting --- streaming-check/movie_check.py | 123 +++++++++++++++++++++------------ 1 file changed, 77 insertions(+), 46 deletions(-) diff --git a/streaming-check/movie_check.py b/streaming-check/movie_check.py index 8cdb1a2..32f1ce2 100644 --- a/streaming-check/movie_check.py +++ b/streaming-check/movie_check.py @@ -6,10 +6,6 @@ from datetime import datetime import os import argparse -def get_args(): - parser = argparse.ArgumentParser(description='Search movie streaming availability.') - parser.add_argument('--year', type=int, help='Specify movie release year') - return parser.parse_args() tmdb_api_token = os.environ.get("TMDB_API_TOKEN") sa_api_token = os.environ.get("SA_API_TOKEN") @@ -27,40 +23,47 @@ sa_headers = { 'x-rapidapi-key': sa_api_token } -args = get_args() -movie = input("Enter a movie: ") -movie_safe = urllib.parse.quote_plus(movie) -if args.year: - tmdb_search = requests.get(f"{tmdb_url}/search/movie?language=en-US&query={movie_safe}&page=1&include_adult=false&primary_release_year={args.year}", headers=tmdb_headers).json() -else: - tmdb_search = requests.get(f"{tmdb_url}/search/movie?language=en-US&query={movie_safe}&page=1&include_adult=false", headers=tmdb_headers).json() - -if not tmdb_search["results"]: - print("I'm having trouble finding that movie. Check your spelling and try again.") - exit() - -movie_id = tmdb_search['results'][0]['id'] -movie_tile = tmdb_search['results'][0]['title'] -movie_release_check = tmdb_search['results'][0]['release_date'] - -if movie_release_check: - movie_release = datetime.strptime(tmdb_search['results'][0]['release_date'], "%Y-%m-%d") -else: movie_release = "???" - -movie_rating = tmdb_search['results'][0]['vote_average'] - -sa_querystring = {"country":"us","tmdb_id":f"movie/{movie_id}","output_language":"en"} -sa_request = requests.request("GET", sa_url, headers=sa_headers, params=sa_querystring) +def get_args(): + parser = argparse.ArgumentParser(description='Search movie streaming availability.') + parser.add_argument('--year', type=int, help='Specify movie release year') + return parser.parse_args() -if sa_request.status_code == 404: - print("I'm having trouble finding that movie. Check your spelling and try again.") - exit() +def tmdb_lookup(tmdb_url, tmdb_headers, tmdb_params): + tmdb_search = requests.get(f"{tmdb_url}/search/movie", params=tmdb_params, headers=tmdb_headers).json() -sa_response = sa_request.json() + if not tmdb_search["results"]: + print("I'm having trouble finding that movie. Check your spelling and try again.") + exit() + + movie_id = tmdb_search['results'][0]['id'] + movie_title = tmdb_search['results'][0]['title'] + movie_release_check = tmdb_search['results'][0]['release_date'] + + if movie_release_check: + movie_release = datetime.strptime(tmdb_search['results'][0]['release_date'], "%Y-%m-%d") + else: movie_release = "???" + + movie_rating = tmdb_search['results'][0]['vote_average'] + + return movie_id,movie_title,movie_release,movie_rating + + +def sa_lookup(sa_url, sa_headers, movie_id): + sa_querystring = {"country":"us","tmdb_id":f"movie/{movie_id}","output_language":"en"} + sa_request = requests.request("GET", sa_url, headers=sa_headers, params=sa_querystring) + + + if sa_request.status_code == 404: + print("I'm having trouble finding that movie. Check your spelling and try again.") + exit() + + sa_response = sa_request.json() + services = sa_response["streamingInfo"] + + return sa_response, services -services = sa_response["streamingInfo"] def services_speller(service): if service == "hbo": @@ -80,17 +83,45 @@ def services_speller(service): else: return service return service_proper - -print(movie_tile + f" ({movie_release.year})") -print(f"Rating: {movie_rating}") -if not services: - print("Streaming not available :(") -for s in services: - leaving_epoch = sa_response["streamingInfo"][s]["us"]["leaving"] - leaving_date = datetime.fromtimestamp(int(leaving_epoch)).strftime('%Y-%m-%d') - link = sa_response["streamingInfo"][s]["us"]["link"] - print(f"Available on {services_speller(s)}") - if leaving_epoch != 0: - print(f"Will be leaving {s} on {leaving_date}") - print(f"Watch here: {link}") \ No newline at end of file + +def main(): + + args = get_args() + movie = input("Enter a movie: ") + movie_safe = urllib.parse.quote_plus(movie) + + tmdb_params = { + "language": "en-US", + "query": movie_safe, + "page": 1, + "include_adult": False + } + + if args.year: + tmdb_params["primary_release_year"] = args.year + + movie_id, movie_title, movie_release, movie_rating = tmdb_lookup(tmdb_url, tmdb_headers, tmdb_params) + + print(movie_title + f" ({movie_release.year})") + print(f"Rating: {movie_rating}\n") + + sa_response, services = sa_lookup(sa_url, sa_headers, movie_id) + + if not services: + print("Streaming not available :(") + + for s in services: + leaving_epoch = sa_response["streamingInfo"][s]["us"]["leaving"] + leaving_date = datetime.fromtimestamp(int(leaving_epoch)).strftime('%Y-%m-%d') + link = sa_response["streamingInfo"][s]["us"]["link"] + + print(f"Available on {services_speller(s)}") + + if leaving_epoch != 0: + print(f"Will be leaving on {leaving_date}") + + print(f"Watch here: {link}\n") + +if __name__ == "__main__": + main() \ No newline at end of file From f4bef215b5ef6930b54c176f2f56d0ff2010c756 Mon Sep 17 00:00:00 2001 From: Ray Lyon Date: Mon, 28 Mar 2022 19:18:31 -0400 Subject: [PATCH 19/24] variable shuffle, remove urllib --- streaming-check/movie_check.py | 64 ++++++++++++++++++++-------------- 1 file changed, 38 insertions(+), 26 deletions(-) diff --git a/streaming-check/movie_check.py b/streaming-check/movie_check.py index 32f1ce2..ce2f3c0 100644 --- a/streaming-check/movie_check.py +++ b/streaming-check/movie_check.py @@ -1,7 +1,6 @@ #!/usr/bin/python3 import requests -import urllib from datetime import datetime import os import argparse @@ -25,16 +24,30 @@ sa_headers = { def get_args(): - parser = argparse.ArgumentParser(description='Search movie streaming availability.') + parser = argparse.ArgumentParser( + description='Search movie streaming availability.') + parser.add_argument('--year', type=int, help='Specify movie release year') return parser.parse_args() -def tmdb_lookup(tmdb_url, tmdb_headers, tmdb_params): - tmdb_search = requests.get(f"{tmdb_url}/search/movie", params=tmdb_params, headers=tmdb_headers).json() +def tmdb_lookup(tmdb_url, tmdb_headers, movie, args): + tmdb_params = { + "language": "en-US", + "query": movie, + "page": 1, + "include_adult": False + } + + if args.year: + tmdb_params["primary_release_year"] = args.year + + tmdb_search = requests.get(f"{tmdb_url}/search/movie", params=tmdb_params, + headers=tmdb_headers).json() if not tmdb_search["results"]: - print("I'm having trouble finding that movie. Check your spelling and try again.") + print("I'm having trouble finding that movie. " + + "Check your spelling and try again.") exit() movie_id = tmdb_search['results'][0]['id'] @@ -42,21 +55,29 @@ def tmdb_lookup(tmdb_url, tmdb_headers, tmdb_params): movie_release_check = tmdb_search['results'][0]['release_date'] if movie_release_check: - movie_release = datetime.strptime(tmdb_search['results'][0]['release_date'], "%Y-%m-%d") - else: movie_release = "???" + movie_release = datetime.strptime( + tmdb_search['results'][0]['release_date'], "%Y-%m-%d") + else: + movie_release = "???" movie_rating = tmdb_search['results'][0]['vote_average'] - return movie_id,movie_title,movie_release,movie_rating + return movie_id, movie_title, movie_release, movie_rating def sa_lookup(sa_url, sa_headers, movie_id): - sa_querystring = {"country":"us","tmdb_id":f"movie/{movie_id}","output_language":"en"} - sa_request = requests.request("GET", sa_url, headers=sa_headers, params=sa_querystring) + sa_params = { + "country": "us", + "tmdb_id": f"movie/{movie_id}", + "output_language": "en" + } + sa_request = requests.request("GET", sa_url, headers=sa_headers, + params=sa_params) if sa_request.status_code == 404: - print("I'm having trouble finding that movie. Check your spelling and try again.") + print("I'm having trouble finding that movie. " + + "Check your spelling and try again.") exit() sa_response = sa_request.json() @@ -89,19 +110,8 @@ def main(): args = get_args() movie = input("Enter a movie: ") - movie_safe = urllib.parse.quote_plus(movie) - tmdb_params = { - "language": "en-US", - "query": movie_safe, - "page": 1, - "include_adult": False - } - - if args.year: - tmdb_params["primary_release_year"] = args.year - - movie_id, movie_title, movie_release, movie_rating = tmdb_lookup(tmdb_url, tmdb_headers, tmdb_params) + movie_id, movie_title, movie_release, movie_rating = tmdb_lookup(tmdb_url, tmdb_headers, movie, args) print(movie_title + f" ({movie_release.year})") print(f"Rating: {movie_rating}\n") @@ -110,10 +120,11 @@ def main(): if not services: print("Streaming not available :(") - + for s in services: leaving_epoch = sa_response["streamingInfo"][s]["us"]["leaving"] - leaving_date = datetime.fromtimestamp(int(leaving_epoch)).strftime('%Y-%m-%d') + leaving_date = datetime.fromtimestamp( + int(leaving_epoch)).strftime('%Y-%m-%d') link = sa_response["streamingInfo"][s]["us"]["link"] print(f"Available on {services_speller(s)}") @@ -123,5 +134,6 @@ def main(): print(f"Watch here: {link}\n") + if __name__ == "__main__": - main() \ No newline at end of file + main() From 2197eaf0648937e0b85c390ae925d68a17d1704e Mon Sep 17 00:00:00 2001 From: Ray Lyon Date: Mon, 28 Mar 2022 19:19:00 -0400 Subject: [PATCH 20/24] initial commit --- .gitignore | 1 + 1 file changed, 1 insertion(+) create mode 100644 .gitignore diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..600d2d3 --- /dev/null +++ b/.gitignore @@ -0,0 +1 @@ +.vscode \ No newline at end of file From 01a3302911c8f5eb18782c03d6eb211996304835 Mon Sep 17 00:00:00 2001 From: Ray Lyon Date: Mon, 28 Mar 2022 19:20:52 -0400 Subject: [PATCH 21/24] rename dir --- {streaming-check => movie-check}/movie_check.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) rename {streaming-check => movie-check}/movie_check.py (97%) diff --git a/streaming-check/movie_check.py b/movie-check/movie_check.py similarity index 97% rename from streaming-check/movie_check.py rename to movie-check/movie_check.py index ce2f3c0..26ea96e 100644 --- a/streaming-check/movie_check.py +++ b/movie-check/movie_check.py @@ -111,7 +111,8 @@ def main(): args = get_args() movie = input("Enter a movie: ") - movie_id, movie_title, movie_release, movie_rating = tmdb_lookup(tmdb_url, tmdb_headers, movie, args) + movie_id, movie_title, movie_release, movie_rating = tmdb_lookup( + tmdb_url, tmdb_headers, movie, args) print(movie_title + f" ({movie_release.year})") print(f"Rating: {movie_rating}\n") From 543fa9734b1bfd53cf0a736b4a5f138eda05ecb0 Mon Sep 17 00:00:00 2001 From: Ray Lyon Date: Mon, 28 Mar 2022 19:30:25 -0400 Subject: [PATCH 22/24] fix for missing date printing --- movie-check/movie_check.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/movie-check/movie_check.py b/movie-check/movie_check.py index 26ea96e..604ee7b 100644 --- a/movie-check/movie_check.py +++ b/movie-check/movie_check.py @@ -57,12 +57,13 @@ def tmdb_lookup(tmdb_url, tmdb_headers, movie, args): if movie_release_check: movie_release = datetime.strptime( tmdb_search['results'][0]['release_date'], "%Y-%m-%d") + movie_year = movie_release.year else: - movie_release = "???" + movie_year = "???" movie_rating = tmdb_search['results'][0]['vote_average'] - return movie_id, movie_title, movie_release, movie_rating + return movie_id, movie_title, movie_year, movie_rating def sa_lookup(sa_url, sa_headers, movie_id): @@ -76,7 +77,7 @@ def sa_lookup(sa_url, sa_headers, movie_id): params=sa_params) if sa_request.status_code == 404: - print("I'm having trouble finding that movie. " + + print("I'm having trouble finding that movie on streaming. " + "Check your spelling and try again.") exit() @@ -114,7 +115,7 @@ def main(): movie_id, movie_title, movie_release, movie_rating = tmdb_lookup( tmdb_url, tmdb_headers, movie, args) - print(movie_title + f" ({movie_release.year})") + print(f"\n{movie_title} ({movie_release})") print(f"Rating: {movie_rating}\n") sa_response, services = sa_lookup(sa_url, sa_headers, movie_id) From 6a9aed30d1f3a43244b8c4e6215ccf26a14dbfb4 Mon Sep 17 00:00:00 2001 From: Ray Lyon Date: Mon, 28 Mar 2022 19:36:08 -0400 Subject: [PATCH 23/24] add tmdb link to result --- movie-check/movie_check.py | 1 + 1 file changed, 1 insertion(+) diff --git a/movie-check/movie_check.py b/movie-check/movie_check.py index 604ee7b..4d66ef5 100644 --- a/movie-check/movie_check.py +++ b/movie-check/movie_check.py @@ -116,6 +116,7 @@ def main(): tmdb_url, tmdb_headers, movie, args) print(f"\n{movie_title} ({movie_release})") + print(f"https://themoviedb.org/movie/{movie_id}") print(f"Rating: {movie_rating}\n") sa_response, services = sa_lookup(sa_url, sa_headers, movie_id) From a765093f643a1769f60f49ecd11c9474c44be65a Mon Sep 17 00:00:00 2001 From: Ray Lyon Date: Mon, 28 Mar 2022 19:48:18 -0400 Subject: [PATCH 24/24] added starz and showtime --- movie-check/movie_check.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/movie-check/movie_check.py b/movie-check/movie_check.py index 4d66ef5..0156327 100644 --- a/movie-check/movie_check.py +++ b/movie-check/movie_check.py @@ -102,6 +102,10 @@ def services_speller(service): service_proper = "Apple TV+" elif service == "paramount": service_proper = "Paramount+" + elif service == "starz": + service_proper = "STARZ" + elif service == "showtime": + service_proper = "Showtime" else: return service return service_proper