YT-DLP Scripting

From Da Nerd Mage Wiki
Jump to navigation Jump to search

YTchan

A script for downloading entire channels...

(assumes your base for downloading YouTube videos is at /mnt/Download_Space/YTDL/ and creates a subfolder using the channel name(s) )

For the potentially required cookie file, see Exporting YouTube cookies 

#!/bin/bash

echo -e "~~~~~~~ " $0 $@ "~~~~~~~ "\\n

if [ $# -eq 0 ] || [ $1 = "-u" ] || [ $1 = "-?" ]; then
  echo -e USAGE: YTchan \[...option\(s\)...\] ChannelName ChannelName ...
  echo
  echo -e OPTIONS:
  echo -e \\t -BOB = Grant Style Folder Structure
  echo
  echo -e \\t -s = simulate this run
  echo -e \\t -c = use a cookie file exported from your browser to identify yourself.
  echo -e \\t -r RATE = Limit bandwidth
  echo -e \\t\\t Maximum download rate in bytes per second,
  echo -e \\t\\t\\t e.g. 50K or 4.2M
  echo -e \\t -p MIN MAX = Sleep interval \(pause\) between downloads \(in seconds\)
  echo -e \\t\\t \(will result in a random pause between MIN \& MAX seconds long\)
  echo -e \\t -e DATE = Earliest video upload date to grab.
  echo -e \\t\\t see https://github.com/yt-dlp/yt-dlp?tab=readme-ov-file#video-selection for format info
  echo -e \\t\\t \(Simplest is YYYYMMDD but fancier options exist\)
  echo -e \\t ChannelName = The name of the channel \(as defined by YouTube\)
  echo -e \\t\\t You can specify multiple channels
  echo
  echo -e The following \"ERROR\"s will also appear in 00000000-ERRORS
  echo -e \(Unfortunately, without indications of which files caused them...\)
  echo
  echo -e  \"ERROR: \[youtube:tab\] \@ChannelName: This channel does not have a videos tab\"
  echo -e  \"ERROR: \[youtube:tab\] \@ChannelName: This channel does not have a shorts tab\"
  echo -e  \"ERROR: \[youtube:tab\] \@ChannelName: This channel does not have a streams tab\"
  echo -e \\t Is simply indicating that the channel doesn\'t have that type of content.
  echo -e \\t \(All 3 means the channel doesn\'t actually have any content... :P \)
  echo
  echo -e \"ERROR: unable to download video data: HTTP Error 403: Forbidden\"
  echo -e \\t Means you may need to supply cookies.
  echo -e \\t Often caused by things like age restrictions.
  echo -e \\t May also be caused by YouTube noticing you\'ve been downloading a bunch...
  echo

  exit
fi

#TYPES="videos"
TYPES="videos shorts streams"
COOKIEfile=/mnt/Download_Space/YTDL/www.youtube.com_cookies.txt

  SIMULATE=""
  WITHcookies=""
  LIMITrate=""
  PAUSEbetween=""
  TIMEspan=""
  BOBstyle=""

while (( "$#" ));
do
  if [ ${1} = "-c" ]; then
    echo Using Cookies: $COOKIEfile
    WITHcookies="--cookies "$COOKIEfile
    shift
  elif [ ${1} = "-s" ]; then
    echo Simulating
    SIMULATE="-s"
    shift
  elif [ ${1} = "-r" ]; then
    shift
    LIMITrate="-r $1"
    shift
    echo Rate Limiting $LIMITrate
  elif [ ${1} = "-p" ]; then
    shift
    Pmin=$1
    shift
    Pmax=$1
    PAUSEbetween="--sleep-interval $Pmin --max-sleep-interval $Pmax"
    shift
    echo Time Span $TIMEspan to current
  elif [ ${1} = "-e" ]; then
    shift
    TIMEspan="--dateafter $1"
    shift
    echo Time Span $TIMEspan to current
  elif [ ${1} = "-BOB" ]; then
    echo Grant Style Folder Structure
    BOBstyle="True"
    shift
  else
    CHANNEL=$1
    shift
    echo
    echo Working on channel: $CHANNEL
    echo ========================================

URL=https://www.youtube.com/\@$CHANNEL

    if wget --spider --quiet $URL > /dev/null 2>&1; then

      DEST=/mnt/Download_Space/YTDL/$CHANNEL

      ERRfile=$DEST/00000000-ERRORS
      ARCHfile=$DEST/11111111-ARCHIVE

      if [ -d $DEST ]; then
        echo $DEST exists
      else
        echo no $DEST... Building it...
        mkdir $DEST
        if [ -z $BOBstyle ]; then
        for ACK in $TYPES
          do
            mkdir $DEST/$ACK
          done
        fi
      fi

      OPTIONS="$SIMULATE $LIMITrate $TIMEspan $WITHcookies $PAUSEbetween"
      OPTIONS+=" --no-overwrites --write-description -t mp4"

      cd $DEST
      echo -e \(storing in $DEST\)
      echo

        for ACK in $TYPES
          do
            echo ">>>>>>>> " $URL/$ACK
            if [ -z $BOBstyle ]; then
              cd $ACK
            fi
            /usr/local/bin/yt-dlp $OPTIONS --download-archive $ARCHfile -o "%(upload_date)s - %(title)s.%(ext)s" $URL/$ACK 2> >(/usr/bin/tee -a $ERRfile)

            # Clear out .part files
            rm -f *.part

            if [ -z $BOBstyle ]; then
              cd ..
            fi
          done

    else
      echo YouTube says this channel does not exist.
    fi   
  fi
done

YTvid

A script for downloading separate videos...

(assumes your base for downloading YouTube videos is at /mnt/Download_Space/YTDL/ and creates a subfolder named _Individual_)

For the potentially required cookie file, see Exporting YouTube cookies 

#!/bin/bash

echo -e "~~~~~~~ " $0 $@ "~~~~~~~ "\\n

if [ $# -eq 0 ] || [ $1 = "-u" ] || [ $1 = "-?" ]; then
  echo -e USAGE: YTvid \[...option\(s\)...\] VideoHash VideoHash ...
  echo
  echo -e OPTIONS:
  echo -e \\t -BOB = Grant Style Folder Structure
  echo
  echo -e \\t -s = simulate this run
  echo -e \\t -c = use a cookie file exported from your browser to identify yourself.
  echo -e \\t -r RATE = Limit bandwidth
  echo -e \\t\\t Maximum download rate in bytes per second,
  echo -e \\t\\t\\t e.g. 50K or 4.2M
  echo -e \\t -p MIN MAX = Sleep interval \(pause\) between downloads \(in seconds\)
  echo -e \\t\\t \(will result in a random pause between MIN \& MAX seconds long\)
  echo -e \\t -e DATE = Earliest video upload date to grab.
  echo -e \\t\\t see https://github.com/yt-dlp/yt-dlp?tab=readme-ov-file#video-selection for format info
  echo -e \\t\\t \(Simplest is YYYYMMDD but fancier options exist\)
  echo -e \\t VideoHash = The name YouTube has given to the individual video
  echo -e \\t\\t \(This is gonna take some work to explain...\)
  echo -e \\t\\t You can specify multiple videos
  echo
  echo -e The following \"ERROR\"s will also appear in 00000000-ERRORS
  echo -e \(Unfortunately, without indications of which files caused them...\)
  echo
  echo -e \"ERROR: unable to download video data: HTTP Error 403: Forbidden\"
  echo -e \\t Means you may need to supply cookies.
  echo -e \\t Often caused by things like age restrictions.
  echo -e \\t May also be caused by YouTube noticing you\'ve been downloading a bunch...
  echo
  echo -e \"ERROR: \[youtube\] oizvS01ovH0: Video unavailable\"
  echo -e \\t Most likely caused by a typo in the VideoHash
  echo

  exit
fi

COOKIEfile=/mnt/Download_Space/YTDL/www.youtube.com_cookies.txt

  SIMULATE=""
  WITHcookies=""
  LIMITrate=""
  PAUSEbetween=""
  TIMEspan=""
  BOBstyle=""

while (( "$#" ));
do
  if [ ${1} = "-c" ]; then
    echo Using Cookies: $COOKIEfile
    WITHcookies="--cookies "$COOKIEfile
    shift
  elif [ ${1} = "-s" ]; then
    echo Simulating
    SIMULATE="-s"
    shift
  elif [ ${1} = "-r" ]; then
    shift
    LIMITrate="-r $1"
    shift
    echo Rate Limiting $LIMITrate
  elif [ ${1} = "-p" ]; then
    shift
    Pmin=$1
    shift
    Pmax=$1
    PAUSEbetween="--sleep-interval $Pmin --max-sleep-interval $Pmax"
    shift
    echo Time Span $TIMEspan to current
  elif [ ${1} = "-e" ]; then
    shift
    TIMEspan="--dateafter $1"
    shift
    echo Time Span $TIMEspan to current
  elif [ ${1} = "-BOB" ]; then
    echo Grant Style Folder Structure
    BOBstyle="True"
    shift
  else
    vidID=$1
    shift
    echo
    echo Working on video: $vidID
    echo ========================================

URL=https://www.youtube.com/watch\?v=$vidID

    if wget --spider --quiet $URL > /dev/null 2>&1; then
    # Weirdly, YouTube says the URL is valid even if it doesn't exist when asking about videos...
    # Then it gives an error while attempting the download.
    # Not fatal, just annoying...

      DEST=/mnt/Download_Space/YTDL/_Individual_/

      if [ -d $DEST ]; then
        echo $DEST exists
      else
        echo no $DEST... Building it...
        mkdir $DEST
      fi

      ERRfile=$DEST/00000000-ERRORS
      ARCHfile=$DEST/11111111-ARCHIVE

      OPTIONS="$SIMULATE $LIMITrate $TIMEspan $WITHcookies $PAUSEbetween"
      OPTIONS+=" --no-overwrites --write-description -t mp4"

      cd $DEST
      echo -e \(storing in $DEST\)
      echo

      /usr/local/bin/yt-dlp $OPTIONS --download-archive $ARCHfile -o "%(uploader_id)s - %(upload_date)s - %(title)s.%(ext)s" $URL/$ACK 2> >(/usr/bin/tee -a $ERRfile)

    else
      echo YouTube says this video does not exist.
    fi   
  fi
done

# Clear out .part files
    rm -f *.part

Gettin' Fancy... One script for both channels & individual videos

WIP

Yup...

Looks even more complicated.

Save this script as YTall

Then make symbolic links to it.

  • ln -s YTall YTc
  • ln -s YTall YTv

Call it as YTc for doing whole channels & YTv for individual videos 

#!/bin/bash

BASENAME=$(basename "$0")

echo -e "~~~~~~~ $BASENAME $@ ~~~~~~~ "\\n

case $BASENAME in
  "YTv")
    TYPE="VideoHash"
    DESC="$TYPE = The name YouTube has given to the individual video"
    DESC+="\n\\t\\t (This is gonna take some work to explain...)"
    DESC+="\n\\t\\t You can specify multiple videos"
    ;;
  "YTc")
    TYPE="ChannelName"
    DESC="$TYPE = The name of the channel \(as defined by YouTube\)"
    DESC+="\n\\t\\t You can specify multiple channels"
    ;;
  *)
    TYPE="poop"
    DESC="$TYPE = Some crap"
    ;;
esac

if [ $# -eq 0 ] || [ $1 = "-u" ] || [ $1 = "-?" ]; then
  echo -e USAGE: $(basename "$0") \[...option\(s\)...\] $TYPE $TYPE ...
  echo
  echo -e OPTIONS:
  echo -e \\t -BOB = Grant Style Folder Structure
  echo
  echo -e \\t -s = simulate this run
  echo -e \\t -c = use a cookie file exported from your browser to identify yourself.
  echo -e \\t -r RATE = Limit bandwidth
  echo -e \\t\\t Maximum download rate in bytes per second,
  echo -e \\t\\t\\t e.g. 50K or 4.2M
  echo -e \\t -p MIN MAX = Sleep interval \(pause\) between downloads \(in seconds\)
  echo -e \\t\\t \(will result in a random pause between MIN \& MAX seconds long\)
  echo -e \\t -e DATE = Earliest video upload date to grab.
  echo -e \\t\\t see https://github.com/yt-dlp/yt-dlp?tab=readme-ov-file#video-selection for format info
  echo -e \\t\\t \(Simplest is YYYYMMDD but fancier options exist\)
  echo -e \\t $DESC
  echo
  echo -e The following \"ERROR\"s will also appear in 00000000-ERRORS
  echo -e \(Unfortunately, without indications of which files caused them...\)
  echo
  echo -e \"ERROR: unable to download video data: HTTP Error 403: Forbidden\"
  echo -e \\t Means you may need to supply cookies.
  echo -e \\t Often caused by things like age restrictions.
  echo -e \\t May also be caused by YouTube noticing you\'ve been downloading a bunch...
case $BASENAME in
  "YTc")
    echo
    echo -e \"ERROR: \[youtube\] oizvS01ovH0: Video unavailable\"
    echo -e \\t Most likely caused by a typo in the VideoHash
  ;;
  "YTv")
    echo
    echo -e  \"ERROR: \[youtube:tab\] \@ChannelName: This channel does not have a videos tab\"
    echo -e  \"ERROR: \[youtube:tab\] \@ChannelName: This channel does not have a shorts tab\"
    echo -e  \"ERROR: \[youtube:tab\] \@ChannelName: This channel does not have a streams tab\"
    echo -e \\t Is simply indicating that the channel doesn\'t have that type of content.
    echo -e \\t \(All 3 means the channel doesn\'t actually have any content... :P \)
  ;;
esac
echo

  exit
fi

  COOKIEfile=/mnt/Download_Space/YTDL/www.youtube.com_cookies.txt

  SIMULATE=""
  WITHcookies=""
  LIMITrate=""
  PAUSEbetween=""
  TIMEspan=""
  BOBstyle=""

while (( "$#" ));
do
  case $1 in
    "-c")
      echo Using Cookies: $COOKIEfile
      WITHcookies="--cookies "$COOKIEfile
      shift
    ;;

    "-s")
      echo Simulating
      SIMULATE="-s"
      shift
    ;;

    "-r")
      LIMITrate="-r $2"
      echo Rate Limiting $2
      shift 2
    ;;

    "-p")
      PAUSEbetween="--sleep-interval $2 --max-sleep-interval $3"
      echo Random pause between $2 \& $3 seconds
      shift 3
    ;;

    "-e")
      TIMEspan="--dateafter $2"
      echo Time Span $2 to current
      shift 2
    ;;

    "-BOB")
      BOBstyle="True"
      echo Using Grant Style Folder Structure
      shift
    ;;

    *)
      case $BASENAME in
        "YTv") # Video
          echo Working on video: $1
          echo ========================================
          URL=https://www.youtube.com/watch\?v=$1
          TYPES=""
          DEST=/mnt/Download_Space/YTDL/_Individual_/
          FORMAT="%(uploader_id)s - %(upload_date)s - %(title)s.%(ext)s"
        ;;

        "YTc") # Channel
          echo Working on Channel: $1
          echo ========================================
          URL=https://www.youtube.com/\@$1
          TYPES="videos shorts streams"
#          TYPES="videos shorts streams playlists"
          DEST=/mnt/Download_Space/YTDL/$1
          FORMAT="%(upload_date)s - %(title)s.%(ext)s"
        ;;
      esac

      ERRfile=$DEST/00000000-ERRORS
      ARCHfile=$DEST/11111111-ARCHIVE

      OPTIONS="$SIMULATE $LIMITrate $TIMEspan $WITHcookies $PAUSEbetween"
      OPTIONS+=" --no-overwrites --write-description -t mp4"

echo $URL

      if wget --spider --quiet $URL > /dev/null 2>&1; then
        # Weirdly, for individual videos, YouTube says the URL is valid even if it doesn't exist when asking about videos...
        # Then it gives an error while attempting the download.
        # Not fatal, just annoying...

        if [ -d $DEST ]; then
          echo $DEST exists
        else
          echo no $DEST... Building it...
          mkdir $DEST
          if [[ $BASENAME = "YTc" && -z $BOBstyle ]]; then
            for ACK in $TYPES
              do
                mkdir $DEST/$ACK
              done
          fi
        fi

        cd $DEST
        echo -en "(storing in $DEST "

        case $BASENAME in
          "YTv")
            echo ")"
            /usr/local/bin/yt-dlp $OPTIONS --download-archive $ARCHfile -o "$FORMAT" $URL 2> >(/usr/bin/tee -a $ERRfile)
            rm -f *.part
          ;;
          "YTc")
            if [ -z $BOBstyle ]; then
              echo -e using subfolders...\)
            else
              echo -e using Grant format...\)
            fi
            echo

            for ACK in $TYPES
              do
                echo ">>>>>>>> " $URL/$ACK
                if [ -z $BOBstyle ]; then
                  cd $ACK
                fi

                /usr/local/bin/yt-dlp $OPTIONS --download-archive $ARCHfile -o "$FORMAT" $URL/$ACK 2> >(/usr/bin/tee -a $ERRfile)

                # Clear out .part files
                rm -f *.part

                if [ -z $BOBstyle ]; then
                  cd ..
                fi
              done
          ;;
        esac
      else
        echo YouTube says this channel does not exist.
      fi   
      shift
    ;;

  esac
done

Even more options?

WIP

Looking into anything special needed for downloading playlists...

Retrieved from ‘https://wiki.nerdmage.ca/index.php?title=YT-DLP_Scripting&oldid=4086’