video & music playlist scripts
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
 
 

111 lines
7.2 KiB

  1. #!/bin/bash
  2. ##this is going to end up becoming a python thing i just know it
  3. set -o pipefail
  4. DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
  5. source ${DIR}/yml_prs.sh
  6. create_variables ${DIR}/config.yaml
  7. #if socket exists, exit. otherwise, create the socket
  8. if [[ -f /tmp/ytmgr.sock ]]; then echo "$(date -Is) - Cron initiated script, but detected socket still exists from a previous run" >> ${ytmgr_log_dir}/ytmgr.log ; exit 1; else touch /tmp/ytmgr.sock; fi
  9. #default dirs&files&logs
  10. mkdir -p ${ytmgr_log_dir}
  11. touch ${ytmgr_log_dir}/success.log ${ytmgr_log_dir}/fail.log
  12. for dir in ${ytmgr_hd_musicvid_dir} ${ytmgr_sd_musicvid_dir} ${ytmgr_channel_dir} ${ytmgr_humor_dir} ${ytmgr_bookmark_dir} ${ytmgr_subscription_dir}; do
  13. mkdir -p ${ytmgr_download_dir}/${dir}
  14. done
  15. /usr/local/bin/youtube-dl -U > /dev/null
  16. ## arg1 is the youtube url arg2 is the destination folder
  17. ytdl_cmd () {
  18. local video_url=$(echo ${1} | cut -d"&" -f1 | sed 's/invidio.us/youtube.com/')
  19. local path=${2}
  20. local country_loop=false
  21. if [[ -z ${video_url} ]]; then echo "$(date -Is) - Invalid line detected for destination ${path}" >> ${ytmgr_log_dir}/ytmgr.log ; exit 1; fi
  22. if [[ ! -d ${path} ]]; then echo "$(date -Is) - Invalid line detected for the path ${path}" >> ${ytmgr_log_dir}/ytmgr.log; exit 1; fi
  23. if [[ -n $(grep "${video_url}" ${ytmgr_log_dir}/success.log) || -n $(grep "${video_url}" ${ytmgr_log_dir}/fail.log) ]]; then return 0; fi
  24. local attempt_count=$(grep "${video_url}" ${ytmgr_log_dir}/ytmgr.log | grep -i "attempt\|fail" | wc -l)
  25. if [[ ${attempt_count} -gt ${ytmgr_attemptsb4_global} ]]; then export country_loop=true; fi
  26. if [[ ${attempt_count} -gt ${ytmgr_max_attempts} ]]; then
  27. echo "$(date -Is) - The video ${video_url} has failed more than ${ytmgr_max_attempts} times. Cancelling its download." >> ${ytmgr_log_dir}/ytmgr.log
  28. echo "${video_url}" >> ${ytmgr_log_dir}/fail.log
  29. elif [[ -n $(echo ${video_url} | grep -v "youtube") ]]; then
  30. echo "$(date -Is) - Trying to download a non-youtube video (${video_url})" >> ${ytmgr_log_dir}/ytmgr.log
  31. if /usr/local/bin/youtube-dl ${video_url} -v --no-playlist -o "${path}/%(uploader)s-%(title)s.%(ext)s" &>> ${ytmgr_log_dir}/ytmgr.log; then
  32. echo "${video_url}" >> ${ytmgr_log_dir}/success.log; echo "$(date -Is) - Video ${video_url} has been downloaded successfully on attempt ${attempt_count}." >> ${ytmgr_log_dir}/ytmgr.log
  33. else export fail_count=$(grep "${video_url}" ${ytmgr_log_dir}/ytmgr.log | grep -i fail | wc -l)
  34. echo "$(date -Is) - Video download of ${video_url} has failed. Failure no. ${fail_count} recorded." >> ${ytmgr_log_dir}/ytmgr.log
  35. fi
  36. else touch ${ytmgr_log_dir}/${video_id}.log
  37. while true; do
  38. local video_id=$(echo "${video_url}" | cut -d"=" -f2)
  39. local attempt_count=$(grep "${video_id}" ${ytmgr_log_dir}/ytmgr.log | grep -i attempt | wc -l)
  40. echo "$(date -Is) - Attempting to download video (${video_url}). Attempt ${attempt_count} to download..." >> ${ytmgr_log_dir}/ytmgr.log
  41. if [[ "${country_loop}" == true ]]; then
  42. for co in ${ytmgr_countries_attempted}; do
  43. echo "$(date -Is) - Attempting to download with country bypass ${co}. Attempt no. ${attempt_count} for video ${video_url}." >> ${ytmgr_log_dir}/ytmgr.log
  44. if /usr/local/bin/youtube-dl ${video_url} -v -4 --no-playlist --geo-bypass --geo-bypass-country ${co} -o "${path}/%(uploader)s_ %(title)s.%(ext)s" &>> ${ytmgr_log_dir}/${video_id}.log; then
  45. echo "${video_url}" >> ${ytmgr_log_dir}/success.log
  46. echo "$(date -Is) - Video ${video_url} has been downloaded successfully on attempt ${attempt_count} to ${dir}." >> ${ytmgr_log_dir}/ytmgr.log
  47. rm -f ${ytmgr_log_dir}/${video_id}.log
  48. break 2
  49. else attempt_count=$((${attempt_count} + 1)); continue
  50. fi
  51. done
  52. break
  53. else if /usr/local/bin/youtube-dl ${video_url} -v -4 --no-playlist -o "${path}/%(uploader)s_ %(title)s.%(ext)s" &>> ${ytmgr_log_dir}/${video_id}.log; then
  54. echo "${video_url}" >> ${ytmgr_log_dir}/success.log
  55. echo "$(date -Is) - Video ${video_url} has been downloaded successfully on attempt ${attempt_count}." >> ${ytmgr_log_dir}/ytmgr.log
  56. rm -f ${ytmgr_log_dir}/${video_id}.log
  57. break
  58. else export fail_count=$(grep -- "$video_id" ${ytmgr_log_dir}/ytmgr.log | grep -i fail | wc -l)
  59. echo "$(date -Is) - Video download of ${video_url} has failed. Failure no. ${fail_count} recorded." >> ${ytmgr_log_dir}/ytmgr.log
  60. if [[ ${fail_count} -le ${ytmgr_attemptsb4_break} ]]; then
  61. export fail_reason="$(grep -i ERROR ${ytmgr_log_dir}/${video_id}.log |tail -n1)"
  62. if [[ -n $(echo ${fail_reason} | grep -i "unavailable\|removed\|private\|unsupported\|not available\|Incomplete") ]]; then
  63. echo "${video_url}" >> ${ytmgr_log_dir}/fail.log
  64. echo "$(date -Is) - The video ${video_url} is not available for download. to search the uri, please click on the following URL: ${global_search}${video_id}" >> ${ytmgr_log_dir}/ytmgr.log
  65. break
  66. elif [[ -n $(echo ${fail_reason} | grep "country\|copyright") ]]; then
  67. export country_loop=true
  68. elif [[ -n $(echo ${fail_reason} | grep "429") ]]; then
  69. echo "$(date -Is) - Got an error 429 for too many http requests. Sleeping for 30 minutes..." >> ${ytmgr_log_dir}/ytmgr.log
  70. sleep 1800
  71. elif [[ -n $(echo ${fail_reason} | grep "urlopen") ]]; then
  72. continue
  73. else echo "$(date -Is) - The video ${video_url} has done something weird to my script or has an uncaught error." >> ${ytmgr_log_dir}/ytmgr.log
  74. break
  75. fi
  76. elif [[ ${fail_count} -le ${ytmgr_attemptsb4_global} ]]; then break
  77. elif [[ ${fail_count} -gt ${ytmgr_attemptsb4_global} ]]; then export country_loop=true
  78. elif [[ ${fail_count} -le ${ytmgr_max_attempts} ]]; then break
  79. else echo "$(date -Is) - Some weird unknown thing happened while trying to download this video, ${video_url}." >> ${ytmgr_log_dir}/ytmgr.log
  80. break
  81. fi
  82. fi
  83. fi
  84. done
  85. fi
  86. sleep 15
  87. }
  88. ##subscriptions
  89. ##turn into function l8r
  90. for video in $(curl -sL "${ytmgr_sub_feed}" | grep youtube.com/watch | cut -d'"' -f2 | grep -v html); do
  91. ytdl_cmd ${video} ${ytmgr_download_dir}/${ytmgr_subscription_dir}/
  92. done
  93. for video in $(curl -sL "${ytmgr_peertube_feed}" | grep -A2 entry | grep href | cut -d'"' -f2); do
  94. ytdl_cmd ${video} ${ytmgr_download_dir}/${ytmgr_subscription_dir}/
  95. done
  96. for video in $(curl -sL "${ytmgr_bookmark_feed}&searchtags=${ytmgr_hd_musicvid_tag}" | grep href | grep content | grep -v "Permalink" | cut -d'"' -f2 | cut -d"&" -f1); do
  97. ytdl_cmd ${video} ${ytmgr_download_dir}/${ytmgr_hd_musicvid_dir}/
  98. done
  99. for video in $(curl -sL "${ytmgr_bookmark_feed}&searchtags=${ytmgr_sd_musicvid_tag}" | grep href | grep content | grep -v "Permalink" | cut -d'"' -f2 | cut -d"&" -f1); do
  100. ytdl_cmd ${video} ${ytmgr_download_dir}/${ytmgr_sd_musicvid_dir}/
  101. done
  102. for video in $(curl -sL "${ytmgr_bookmark_feed}&searchtags=${ytmgr_humor_tag}" | grep href | grep content | grep -v "Permalink" | cut -d'"' -f2 | cut -d"&" -f1); do
  103. ytdl_cmd ${video} ${ytmgr_download_dir}/${ytmgr_humor_dir}/
  104. done
  105. for video in $(curl -sL "${ytmgr_bookmark_feed}" | grep href | grep content | grep -v "Permalink" | cut -d'"' -f2 | cut -d"&" -f1); do
  106. ytdl_cmd ${video} ${ytmgr_download_dir}/${ytmgr_bookmark_dir}/
  107. done
  108. rm -f /tmp/ytmgr.sock