keyplays/ytmgr.sh

#!/bin/bash
##this is going to end up becoming a python thing i just know it
set -o pipefail
DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
source ${DIR}/yml_prs.sh
create_variables ${DIR}/config.yaml
#if socket exists, exit. otherwise, create the socket
if [[ -f /tmp/ytmgr.sock ]]; then echo "$(date -Is) - Cron initiated script, but detected socket still exists from a previous run" >> ${ytmgr_log_dir}/ytmgr.log ; exit 1; else touch /tmp/ytmgr.sock; fi
#default dirs&files&logs
mkdir -p ${ytmgr_log_dir}
touch ${ytmgr_log_dir}/success.log ${ytmgr_log_dir}/fail.log
for dir in ${ytmgr_hd_musicvid_dir} ${ytmgr_sd_musicvid_dir} ${ytmgr_channel_dir} ${ytmgr_humor_dir} ${ytmgr_bookmark_dir} ${ytmgr_subscription_dir}; do
	mkdir -p ${ytmgr_download_dir}/${dir}
done
/usr/local/bin/youtube-dl -U > /dev/null
## arg1 is the youtube url arg2 is the destination folder
ytdl_cmd () {
	local video_url=$(echo ${1} | cut -d"&" -f1 | sed 's/invidio.us/youtube.com/')
	local path=${2}
	local country_loop=false
	if [[ -z ${video_url} ]]; then echo "$(date -Is) - Invalid line detected for destination ${path}" >> ${ytmgr_log_dir}/ytmgr.log ; exit 1; fi
	if [[ ! -d ${path} ]]; then echo "$(date -Is) - Invalid line detected for the path ${path}" >> ${ytmgr_log_dir}/ytmgr.log; exit 1; fi
	if [[ -n $(grep "${video_url}" ${ytmgr_log_dir}/success.log) || -n $(grep "${video_url}" ${ytmgr_log_dir}/fail.log) ]]; then return 0; fi
	local attempt_count=$(grep "${video_url}" ${ytmgr_log_dir}/ytmgr.log | grep -i "attempt\|fail" | wc -l)
	if [[ ${attempt_count} -gt ${ytmgr_attemptsb4_global} ]]; then export country_loop=true; fi
	if [[ ${attempt_count} -gt ${ytmgr_max_attempts} ]]; then
		echo "$(date -Is) - The video ${video_url} has failed more than ${ytmgr_max_attempts} times. Cancelling its download." >> ${ytmgr_log_dir}/ytmgr.log
		echo "${video_url}" >> ${ytmgr_log_dir}/fail.log
	elif [[ -n $(echo ${video_url} | grep -v "youtube") ]]; then
		echo "$(date -Is) - Trying to download a non-youtube video (${video_url})" >> ${ytmgr_log_dir}/ytmgr.log
		if /usr/local/bin/youtube-dl ${video_url} -v --no-playlist -o "${path}/%(uploader)s-%(title)s.%(ext)s" &>> ${ytmgr_log_dir}/ytmgr.log; then
		        echo "${video_url}" >> ${ytmgr_log_dir}/success.log; echo "$(date -Is) - Video ${video_url} has been downloaded successfully on attempt ${attempt_count}." >> ${ytmgr_log_dir}/ytmgr.log
		else export fail_count=$(grep "${video_url}" ${ytmgr_log_dir}/ytmgr.log | grep -i fail | wc -l)
                     echo "$(date -Is) - Video download of ${video_url} has failed. Failure no. ${fail_count} recorded."  >> ${ytmgr_log_dir}/ytmgr.log
		fi
	else touch ${ytmgr_log_dir}/${video_id}.log
		while true; do
			local video_id=$(echo "${video_url}" | cut -d"=" -f2)
			local attempt_count=$(grep "${video_id}" ${ytmgr_log_dir}/ytmgr.log | grep -i attempt | wc -l)
			echo "$(date -Is) - Attempting to download video (${video_url}). Attempt ${attempt_count} to download..." >> ${ytmgr_log_dir}/ytmgr.log
			if [[ "${country_loop}" == true ]]; then
				for co in ${ytmgr_countries_attempted}; do
					echo "$(date -Is) - Attempting to download with country bypass ${co}. Attempt no. ${attempt_count} for video ${video_url}." >> ${ytmgr_log_dir}/ytmgr.log
					if /usr/local/bin/youtube-dl ${video_url} -v -4  --no-playlist --geo-bypass --geo-bypass-country ${co} -o "${path}/%(uploader)s_ %(title)s.%(ext)s" &>> ${ytmgr_log_dir}/${video_id}.log; then
						echo "${video_url}" >> ${ytmgr_log_dir}/success.log
						echo "$(date -Is) - Video ${video_url} has been downloaded successfully on attempt ${attempt_count} to ${dir}." >> ${ytmgr_log_dir}/ytmgr.log
						rm -f ${ytmgr_log_dir}/${video_id}.log
						break 2
					else attempt_count=$((${attempt_count} + 1)); continue
					fi
				done
				break

			else	if /usr/local/bin/youtube-dl ${video_url} -v -4 --no-playlist -o "${path}/%(uploader)s_ %(title)s.%(ext)s" &>> ${ytmgr_log_dir}/${video_id}.log; then
						echo "${video_url}" >> ${ytmgr_log_dir}/success.log
						echo "$(date -Is) - Video ${video_url} has been downloaded successfully on attempt ${attempt_count}." >> ${ytmgr_log_dir}/ytmgr.log
						rm -f ${ytmgr_log_dir}/${video_id}.log
						break
					else export fail_count=$(grep -- "$video_id" ${ytmgr_log_dir}/ytmgr.log | grep -i fail | wc -l)
						echo "$(date -Is) - Video download of ${video_url} has failed. Failure no. ${fail_count} recorded."  >> ${ytmgr_log_dir}/ytmgr.log
						if [[ ${fail_count} -le ${ytmgr_attemptsb4_break} ]]; then
							export fail_reason="$(grep -i ERROR ${ytmgr_log_dir}/${video_id}.log |tail -n1)" 
							if [[ -n $(echo ${fail_reason} | grep -i "unavailable\|removed\|private\|unsupported\|not available\|Incomplete") ]]; then
								echo "${video_url}" >> ${ytmgr_log_dir}/fail.log
								echo "$(date -Is) - The video ${video_url} is not available for download. to search the uri, please click on the following URL: ${global_search}${video_id}" >> ${ytmgr_log_dir}/ytmgr.log
								break
							elif [[ -n $(echo ${fail_reason} | grep "country\|copyright") ]]; then
								export country_loop=true
							elif [[ -n $(echo ${fail_reason} | grep "429") ]]; then
								echo "$(date -Is) - Got an error 429 for too many http requests. Sleeping for 30 minutes..." >> ${ytmgr_log_dir}/ytmgr.log
								sleep 1800
							elif [[ -n $(echo ${fail_reason} | grep "urlopen") ]]; then
								continue
							else echo "$(date -Is) - The video ${video_url} has done something weird to my script or has an uncaught error." >> ${ytmgr_log_dir}/ytmgr.log
								break
							fi
						elif [[ ${fail_count} -le ${ytmgr_attemptsb4_global} ]]; then break
						elif [[ ${fail_count} -gt ${ytmgr_attemptsb4_global} ]]; then export country_loop=true
						elif [[ ${fail_count} -le ${ytmgr_max_attempts} ]]; then break
						else echo "$(date -Is) - Some weird unknown thing happened while trying to download this video, ${video_url}." >> ${ytmgr_log_dir}/ytmgr.log
							break
						fi
					fi
			fi
		done
	fi
	sleep 15
}
##subscriptions

##turn into function l8r
for video in $(curl -sL "${ytmgr_sub_feed}" | grep youtube.com/watch | cut -d'"' -f2 | grep -v html); do
	ytdl_cmd ${video} ${ytmgr_download_dir}/${ytmgr_subscription_dir}/
done
for video in $(curl -sL "${ytmgr_peertube_feed}" | grep -A2 entry | grep href | cut -d'"' -f2); do
        ytdl_cmd ${video} ${ytmgr_download_dir}/${ytmgr_subscription_dir}/
done
for video in $(curl -sL "${ytmgr_bookmark_feed}&searchtags=${ytmgr_hd_musicvid_tag}" | grep href | grep content | grep -v "Permalink" | cut -d'"' -f2 | cut -d"&" -f1); do
	ytdl_cmd ${video} ${ytmgr_download_dir}/${ytmgr_hd_musicvid_dir}/
done
for video in $(curl -sL "${ytmgr_bookmark_feed}&searchtags=${ytmgr_sd_musicvid_tag}" | grep href | grep content | grep -v "Permalink" | cut -d'"' -f2 | cut -d"&" -f1); do
	ytdl_cmd ${video} ${ytmgr_download_dir}/${ytmgr_sd_musicvid_dir}/
done
for video in $(curl -sL "${ytmgr_bookmark_feed}&searchtags=${ytmgr_humor_tag}" | grep href | grep content | grep -v "Permalink" | cut -d'"' -f2 | cut -d"&" -f1); do
	ytdl_cmd ${video} ${ytmgr_download_dir}/${ytmgr_humor_dir}/
done
for video in $(curl -sL "${ytmgr_bookmark_feed}" | grep href | grep content | grep -v "Permalink" | cut -d'"' -f2 | cut -d"&" -f1); do
	ytdl_cmd ${video} ${ytmgr_download_dir}/${ytmgr_bookmark_dir}/
done
rm -f /tmp/ytmgr.sock