From 0c4eec30b86f2d5c08d1d89bf299672b16b1e2af Mon Sep 17 00:00:00 2001 From: HaarigerHarald Date: Fri, 1 Apr 2016 18:06:12 +0200 Subject: [PATCH 1/4] Extract extra video meta data author, video length, view count, ... --- README.md | 8 +- .../youtubeDownloader/DownloadActivity.java | 31 ++- .../DownloadFinishedReceiver.java | 7 +- .../SampleDownloadActivity.java | 34 ++- .../huber/youtubeExtractor/ExtractorCase.java | 46 ++-- .../{Meta.java => Format.java} | 12 +- .../at/huber/youtubeExtractor/VideoMeta.java | 77 ++++++ ...riExtractor.java => YouTubeExtractor.java} | 222 +++++++++++------- .../at/huber/youtubeExtractor/YtFile.java | 12 +- 9 files changed, 281 insertions(+), 168 deletions(-) rename youtubeExtractor/src/main/java/at/huber/youtubeExtractor/{Meta.java => Format.java} (81%) create mode 100644 youtubeExtractor/src/main/java/at/huber/youtubeExtractor/VideoMeta.java rename youtubeExtractor/src/main/java/at/huber/youtubeExtractor/{YouTubeUriExtractor.java => YouTubeExtractor.java} (75%) diff --git a/README.md b/README.md index 974f622..65f385e 100644 --- a/README.md +++ b/README.md @@ -31,17 +31,15 @@ It's build around an AsyncTask. Called from an Activity you can write: ```java String youtubeLink = "http://youtube.com/watch?v=xxxx"; -YouTubeUriExtractor ytEx = new YouTubeUriExtractor(this) { +new YouTubeExtractor(this) { @Override - public void onUrisAvailable(String videoId, String videoTitle, SparseArray ytFiles) { + public void onExtractionComplete(SparseArray ytFiles, VideoMeta vMeta) { if (ytFiles != null) { int itag = 22; String downloadUrl = ytFiles.get(itag).getUrl(); } } -}; - -ytEx.execute(youtubeLink); +}.extract(youtubeLink, true, true); ``` The ytFiles SparseArray is a map of available media files for one YouTube video, accessible by their itag diff --git a/advancedDownloader/src/main/java/at/huber/youtubeDownloader/DownloadActivity.java b/advancedDownloader/src/main/java/at/huber/youtubeDownloader/DownloadActivity.java index c9be187..3e32458 100644 --- a/advancedDownloader/src/main/java/at/huber/youtubeDownloader/DownloadActivity.java +++ b/advancedDownloader/src/main/java/at/huber/youtubeDownloader/DownloadActivity.java @@ -24,7 +24,8 @@ import java.util.Comparator; import java.util.List; -import at.huber.youtubeExtractor.YouTubeUriExtractor; +import at.huber.youtubeExtractor.VideoMeta; +import at.huber.youtubeExtractor.YouTubeExtractor; import at.huber.youtubeExtractor.YtFile; public class DownloadActivity extends Activity { @@ -68,10 +69,10 @@ && getIntent().getType() != null && "text/plain".equals(getIntent().getType())) } private void getYoutubeDownloadUrl(String youtubeLink) { - YouTubeUriExtractor ytEx = new YouTubeUriExtractor(this) { + new YouTubeExtractor(this) { @Override - public void onUrisAvailable(String videoId, String videoTitle, SparseArray ytFiles) { + public void onExtractionComplete(SparseArray ytFiles, VideoMeta vMeta) { mainProgressBar.setVisibility(View.GONE); if (ytFiles == null) { TextView tv = new TextView(DownloadActivity.this); @@ -85,7 +86,7 @@ public void onUrisAvailable(String videoId, String videoTitle, SparseArray= 360) { + if (ytFile.getFormat().getHeight() == -1 || ytFile.getFormat().getHeight() >= 360) { addFormatToList(ytFile, ytFiles); } } @@ -96,29 +97,25 @@ public int compare(YtFragmentedVideo lhs, YtFragmentedVideo rhs) { } }); for (YtFragmentedVideo files : formatsToShowList) { - addButtonToMainLayout(videoTitle, files); + addButtonToMainLayout(vMeta.getTitle(), files); } } - }; - ytEx.setIncludeWebM(false); - ytEx.setParseDashManifest(true); - ytEx.execute(youtubeLink); - + }.extract(youtubeLink, true, false); } private void addFormatToList(YtFile ytFile, SparseArray ytFiles) { - int height = ytFile.getMeta().getHeight(); + int height = ytFile.getFormat().getHeight(); if (height != -1) { for (YtFragmentedVideo frVideo : formatsToShowList) { if (frVideo.height == height && (frVideo.videoFile == null || - frVideo.videoFile.getMeta().getFps() == ytFile.getMeta().getFps())) { + frVideo.videoFile.getFormat().getFps() == ytFile.getFormat().getFps())) { return; } } } YtFragmentedVideo frVideo = new YtFragmentedVideo(); frVideo.height = height; - if (ytFile.getMeta().isDashContainer()) { + if (ytFile.getFormat().isDashContainer()) { if (height > 0) { frVideo.videoFile = ytFile; frVideo.audioFile = ytFiles.get(ITAG_FOR_AUDIO); @@ -136,9 +133,9 @@ private void addButtonToMainLayout(final String videoTitle, final YtFragmentedVi // Display some buttons and let the user choose the format String btnText; if (ytFrVideo.height == -1) - btnText = "Audio " + ytFrVideo.audioFile.getMeta().getAudioBitrate() + " kbit/s"; + btnText = "Audio " + ytFrVideo.audioFile.getFormat().getAudioBitrate() + " kbit/s"; else - btnText = (ytFrVideo.videoFile.getMeta().getFps() == 60) ? ytFrVideo.height + "p60" : + btnText = (ytFrVideo.videoFile.getFormat().getFps() == 60) ? ytFrVideo.height + "p60" : ytFrVideo.height + "p"; Button btn = new Button(this); btn.setText(btnText); @@ -158,13 +155,13 @@ public void onClick(View v) { boolean hideAudioDownloadNotification = false; if (ytFrVideo.videoFile != null) { downloadIds += downloadFromUrl(ytFrVideo.videoFile.getUrl(), videoTitle, - filename + "." + ytFrVideo.videoFile.getMeta().getExt(), false); + filename + "." + ytFrVideo.videoFile.getFormat().getExt(), false); downloadIds += "-"; hideAudioDownloadNotification = true; } if (ytFrVideo.audioFile != null) { downloadIds += downloadFromUrl(ytFrVideo.audioFile.getUrl(), videoTitle, - filename + "." + ytFrVideo.audioFile.getMeta().getExt(), hideAudioDownloadNotification); + filename + "." + ytFrVideo.audioFile.getFormat().getExt(), hideAudioDownloadNotification); } if (ytFrVideo.audioFile != null) cacheDownloadIds(downloadIds); diff --git a/advancedDownloader/src/main/java/at/huber/youtubeDownloader/DownloadFinishedReceiver.java b/advancedDownloader/src/main/java/at/huber/youtubeDownloader/DownloadFinishedReceiver.java index 3163a9d..1a4aa2c 100644 --- a/advancedDownloader/src/main/java/at/huber/youtubeDownloader/DownloadFinishedReceiver.java +++ b/advancedDownloader/src/main/java/at/huber/youtubeDownloader/DownloadFinishedReceiver.java @@ -43,7 +43,7 @@ public class DownloadFinishedReceiver extends BroadcastReceiver { private static final String TEMP_FILE_NAME = "tmp-"; private static final Pattern ARTIST_TITLE_PATTERN = - Pattern.compile("(.+?)(\\s*?)-(\\s*?)(\"|)(\\S(.+?))\\s*?([\\&\\*+,-/:;<=>@_\\|]+?\\s*?|)(\\z|\"|\\(|\\[|lyric|official)", + Pattern.compile("(.+?)(\\s*?)-(\\s*?)(\"|)(\\S(.+?))\\s*?([&\\*+,-/:;<=>@_\\|]+?\\s*?|)(\\z|\"|\\(|\\[|lyric|official)", Pattern.CASE_INSENSITIVE | Pattern.UNICODE_CASE); @Override @@ -202,8 +202,7 @@ private void writeMetaData(Container out, String artist, String title) { } } if (mBox != null) { - UserDataBox userDataBox = null; - userDataBox = new UserDataBox(); + UserDataBox userDataBox = new UserDataBox(); mBox.addBox(userDataBox); MetaBox metaBox = new MetaBox(); userDataBox.addBox(metaBox); @@ -231,7 +230,7 @@ private void correctChunkOffsets(Container container, long correction) { List chunkOffsetBoxes = Path.getPaths(container, "/moov[0]/trak/mdia[0]/minf[0]/stbl[0]/stco[0]"); for (Box chunkOffsetBox : chunkOffsetBoxes) { - LinkedList stblChildren = new LinkedList(chunkOffsetBox.getParent().getBoxes()); + LinkedList stblChildren = new LinkedList<>(chunkOffsetBox.getParent().getBoxes()); stblChildren.remove(chunkOffsetBox); long[] cOffsets = ((ChunkOffsetBox) chunkOffsetBox).getChunkOffsets(); diff --git a/sampleApp/src/main/java/at/huber/sampleDownload/SampleDownloadActivity.java b/sampleApp/src/main/java/at/huber/sampleDownload/SampleDownloadActivity.java index 353a2b7..df99816 100644 --- a/sampleApp/src/main/java/at/huber/sampleDownload/SampleDownloadActivity.java +++ b/sampleApp/src/main/java/at/huber/sampleDownload/SampleDownloadActivity.java @@ -15,7 +15,8 @@ import android.widget.ProgressBar; import android.widget.Toast; -import at.huber.youtubeExtractor.YouTubeUriExtractor; +import at.huber.youtubeExtractor.VideoMeta; +import at.huber.youtubeExtractor.YouTubeExtractor; import at.huber.youtubeExtractor.YtFile; public class SampleDownloadActivity extends Activity { @@ -56,10 +57,10 @@ && getIntent().getType() != null && "text/plain".equals(getIntent().getType())) } private void getYoutubeDownloadUrl(String youtubeLink) { - YouTubeUriExtractor ytEx = new YouTubeUriExtractor(this) { + new YouTubeExtractor(this) { @Override - public void onUrisAvailable(String videoId, String videoTitle, SparseArray ytFiles) { + public void onExtractionComplete(SparseArray ytFiles, VideoMeta vMeta) { mainProgressBar.setVisibility(View.GONE); if (ytFiles == null) { @@ -68,32 +69,26 @@ public void onUrisAvailable(String videoId, String videoTitle, SparseArray height -1 = audio - if (ytFile.getMeta().getHeight() == -1 || ytFile.getMeta().getHeight() >= 360) { - addButtonToMainLayout(videoTitle, ytFile); + if (ytFile.getFormat().getHeight() == -1 || ytFile.getFormat().getHeight() >= 360) { + addButtonToMainLayout(vMeta.getTitle(), ytFile); } } } - }; - // Ignore the webm container format - ytEx.setIncludeWebM(false); - ytEx.setParseDashManifest(true); - // Lets execute the request - ytEx.execute(youtubeLink); - + }.extract(youtubeLink, true, false); } private void addButtonToMainLayout(final String videoTitle, final YtFile ytfile) { // Display some buttons and let the user choose the format - String btnText = (ytfile.getMeta().getHeight() == -1) ? "Audio " + - ytfile.getMeta().getAudioBitrate() + " kbit/s" : - ytfile.getMeta().getHeight() + "p"; - btnText += (ytfile.getMeta().isDashContainer()) ? " dash" : ""; + String btnText = (ytfile.getFormat().getHeight() == -1) ? "Audio " + + ytfile.getFormat().getAudioBitrate() + " kbit/s" : + ytfile.getFormat().getHeight() + "p"; + btnText += (ytfile.getFormat().isDashContainer()) ? " dash" : ""; Button btn = new Button(this); btn.setText(btnText); btn.setOnClickListener(new OnClickListener() { @@ -102,9 +97,9 @@ private void addButtonToMainLayout(final String videoTitle, final YtFile ytfile) public void onClick(View v) { String filename; if (videoTitle.length() > 55) { - filename = videoTitle.substring(0, 55) + "." + ytfile.getMeta().getExt(); + filename = videoTitle.substring(0, 55) + "." + ytfile.getFormat().getExt(); } else { - filename = videoTitle + "." + ytfile.getMeta().getExt(); + filename = videoTitle + "." + ytfile.getFormat().getExt(); } filename = filename.replaceAll("\\\\|>|<|\"|\\||\\*|\\?|%|:|#|/", ""); downloadFromUrl(ytfile.getUrl(), videoTitle, filename); @@ -123,7 +118,6 @@ private void downloadFromUrl(String youtubeDlUrl, String downloadTitle, String f request.setNotificationVisibility(DownloadManager.Request.VISIBILITY_VISIBLE_NOTIFY_COMPLETED); request.setDestinationInExternalPublicDir(Environment.DIRECTORY_DOWNLOADS, fileName); - // get download service and enqueue file DownloadManager manager = (DownloadManager) getSystemService(Context.DOWNLOAD_SERVICE); manager.enqueue(request); } diff --git a/youtubeExtractor/src/androidTest/java/at/huber/youtubeExtractor/ExtractorCase.java b/youtubeExtractor/src/androidTest/java/at/huber/youtubeExtractor/ExtractorCase.java index 66979d8..aeb6394 100644 --- a/youtubeExtractor/src/androidTest/java/at/huber/youtubeExtractor/ExtractorCase.java +++ b/youtubeExtractor/src/androidTest/java/at/huber/youtubeExtractor/ExtractorCase.java @@ -10,9 +10,6 @@ import android.util.Log; import android.util.SparseArray; -import at.huber.youtubeExtractor.YouTubeUriExtractor; -import at.huber.youtubeExtractor.YtFile; - public class ExtractorCase extends InstrumentationTestCase { private static final String EXTRACTOR_TEST_TAG = "Extractor Test"; @@ -20,42 +17,48 @@ public class ExtractorCase extends InstrumentationTestCase { private String testUrl; public void testUsualVideo() throws Throwable { - extractorTest("http://youtube.com/watch?v=YE7VzlLtp-4", "YE7VzlLtp-4", "Big Buck Bunny"); + VideoMeta expMeta = new VideoMeta("YE7VzlLtp-4", "Big Buck Bunny", "Blender Foundation", + "UCSMOQeBJ2RAnuFungnQOxLg", 597, 0); + extractorTest("http://youtube.com/watch?v=YE7VzlLtp-4", expMeta); extractorTestDashManifest("http://youtube.com/watch?v=YE7VzlLtp-4"); } public void testEncipheredVideo() throws Throwable { - extractorTest("https://www.youtube.com/watch?v=e8X3ACToii0", "e8X3ACToii0", - "Rise Against - Savior"); + VideoMeta expMeta = new VideoMeta("e8X3ACToii0", "Rise Against - Savior", "RiseAgainstVEVO", + "UChMKB2AHNpeuWhalpRYhUaw", 244, 0); + extractorTest("https://www.youtube.com/watch?v=e8X3ACToii0", expMeta); extractorTestDashManifest("https://www.youtube.com/watch?v=e8X3ACToii0"); } public void testAgeRestrictVideo() throws Throwable { - extractorTest("http://www.youtube.com/watch?v=61Ev-YvBw2c", "61Ev-YvBw2c", - "Test video for age-restriction"); + VideoMeta expMeta = new VideoMeta("61Ev-YvBw2c", "Test video for age-restriction", + "jpdemoA", "UC95NqtFsDZKlmzOJmZi_g6Q", 14, 0); + extractorTest("http://www.youtube.com/watch?v=61Ev-YvBw2c", expMeta); extractorTestDashManifest("http://www.youtube.com/watch?v=61Ev-YvBw2c"); } private void extractorTestDashManifest(final String youtubeLink) throws Throwable { final CountDownLatch signal = new CountDownLatch(1); + YouTubeExtractor.LOGGING = true; + testUrl = null; runTestOnUiThread(new Runnable() { @Override public void run() { - final YouTubeUriExtractor ytEx = new YouTubeUriExtractor(getInstrumentation() + final YouTubeExtractor ytEx = new YouTubeExtractor(getInstrumentation() .getTargetContext()) { @Override - public void onUrisAvailable(String videoId, String videoTitle, SparseArray ytFiles) { + public void onExtractionComplete(SparseArray ytFiles, VideoMeta videoMeta) { assertNotNull(ytFiles); int numNotDash = 0; int itag; for (int i = 0; i < ytFiles.size(); i++) { itag = ytFiles.keyAt(i); - if (ytFiles.get(itag).getMeta().isDashContainer()) { + if (ytFiles.get(itag).getFormat().isDashContainer()) { numNotDash = i; break; } @@ -66,8 +69,7 @@ public void onUrisAvailable(String videoId, String videoTitle, SparseArray ytFiles) { - assertEquals(videoId, expVideoId); - assertEquals(videoTitle, expVideoTitle); + public void onExtractionComplete(SparseArray ytFiles, VideoMeta videoMeta) { + assertEquals(expMeta.getVideoId(), videoMeta.getVideoId()); + assertEquals(expMeta.getTitle(),videoMeta.getTitle()); + assertEquals(expMeta.getAuthor(), videoMeta.getAuthor()); + assertEquals(expMeta.getChannelId(), videoMeta.getChannelId()); + assertEquals(expMeta.getVideoLength(), videoMeta.getVideoLength()); + assertNotSame(0, videoMeta.getViewCount()); assertNotNull(ytFiles); int itag = ytFiles.keyAt(new Random().nextInt(ytFiles.size())); Log.d(EXTRACTOR_TEST_TAG, "Testing itag:" + itag); @@ -106,7 +114,7 @@ public void onUrisAvailable(String videoId, String videoTitle, SparseArray> { -import com.evgenii.jsevaluator.JsEvaluator; -import com.evgenii.jsevaluator.interfaces.JsCallback; + private final static boolean CACHING = true; -public abstract class YouTubeUriExtractor extends AsyncTask> { + protected static boolean LOGGING = false; - private final static boolean CACHING = true; - private final static boolean LOGGING = false; - private final static String LOG_TAG = "YouTubeUriExtractor"; + private final static String LOG_TAG = "YouTubeExtractor"; private final static String CACHE_FILE_NAME = "decipher_js_funct"; private final static int DASH_PARSE_RETRIES = 5; private Context context; - private String videoTitle; - private String youtubeID; + private String videoID; + private VideoMeta videoMeta; private boolean includeWebM = true; private boolean useHttp = false; private boolean parseDashManifest = false; @@ -62,6 +65,12 @@ public abstract class YouTubeUriExtractor extends AsyncTask META_MAP = new SparseArray<>(); + private static final SparseArray FORMAT_MAP = new SparseArray<>(); static { // http://en.wikipedia.org/wiki/YouTube#Quality_and_formats // Video and Audio - META_MAP.put(17, new Meta(17, "3gp", 144, Meta.VCodec.MPEG4, Meta.ACodec.AAC, 24, false)); - META_MAP.put(36, new Meta(36, "3gp", 240, Meta.VCodec.MPEG4, Meta.ACodec.AAC, 32, false)); - META_MAP.put(5, new Meta(5, "flv", 240, Meta.VCodec.H263, Meta.ACodec.MP3, 64, false)); - META_MAP.put(43, new Meta(43, "webm", 360, Meta.VCodec.VP8, Meta.ACodec.VORBIS, 128, false)); - META_MAP.put(18, new Meta(18, "mp4", 360, Meta.VCodec.H264, Meta.ACodec.AAC, 96, false)); - META_MAP.put(22, new Meta(22, "mp4", 720, Meta.VCodec.H264, Meta.ACodec.AAC, 192, false)); + FORMAT_MAP.put(17, new Format(17, "3gp", 144, Format.VCodec.MPEG4, Format.ACodec.AAC, 24, false)); + FORMAT_MAP.put(36, new Format(36, "3gp", 240, Format.VCodec.MPEG4, Format.ACodec.AAC, 32, false)); + FORMAT_MAP.put(5, new Format(5, "flv", 240, Format.VCodec.H263, Format.ACodec.MP3, 64, false)); + FORMAT_MAP.put(43, new Format(43, "webm", 360, Format.VCodec.VP8, Format.ACodec.VORBIS, 128, false)); + FORMAT_MAP.put(18, new Format(18, "mp4", 360, Format.VCodec.H264, Format.ACodec.AAC, 96, false)); + FORMAT_MAP.put(22, new Format(22, "mp4", 720, Format.VCodec.H264, Format.ACodec.AAC, 192, false)); // Dash Video - META_MAP.put(160, new Meta(160, "mp4", 144, Meta.VCodec.H264, Meta.ACodec.NONE, true)); - META_MAP.put(133, new Meta(133, "mp4", 240, Meta.VCodec.H264, Meta.ACodec.NONE, true)); - META_MAP.put(134, new Meta(134, "mp4", 360, Meta.VCodec.H264, Meta.ACodec.NONE, true)); - META_MAP.put(135, new Meta(135, "mp4", 480, Meta.VCodec.H264, Meta.ACodec.NONE, true)); - META_MAP.put(136, new Meta(136, "mp4", 720, Meta.VCodec.H264, Meta.ACodec.NONE, true)); - META_MAP.put(137, new Meta(137, "mp4", 1080, Meta.VCodec.H264, Meta.ACodec.NONE, true)); - META_MAP.put(264, new Meta(264, "mp4", 1440, Meta.VCodec.H264, Meta.ACodec.NONE, true)); - META_MAP.put(266, new Meta(266, "mp4", 2160, Meta.VCodec.H264, Meta.ACodec.NONE, true)); - - META_MAP.put(298, new Meta(298, "mp4", 720, Meta.VCodec.H264, 60, Meta.ACodec.NONE, true)); - META_MAP.put(299, new Meta(299, "mp4", 1080, Meta.VCodec.H264, 60, Meta.ACodec.NONE, true)); + FORMAT_MAP.put(160, new Format(160, "mp4", 144, Format.VCodec.H264, Format.ACodec.NONE, true)); + FORMAT_MAP.put(133, new Format(133, "mp4", 240, Format.VCodec.H264, Format.ACodec.NONE, true)); + FORMAT_MAP.put(134, new Format(134, "mp4", 360, Format.VCodec.H264, Format.ACodec.NONE, true)); + FORMAT_MAP.put(135, new Format(135, "mp4", 480, Format.VCodec.H264, Format.ACodec.NONE, true)); + FORMAT_MAP.put(136, new Format(136, "mp4", 720, Format.VCodec.H264, Format.ACodec.NONE, true)); + FORMAT_MAP.put(137, new Format(137, "mp4", 1080, Format.VCodec.H264, Format.ACodec.NONE, true)); + FORMAT_MAP.put(264, new Format(264, "mp4", 1440, Format.VCodec.H264, Format.ACodec.NONE, true)); + FORMAT_MAP.put(266, new Format(266, "mp4", 2160, Format.VCodec.H264, Format.ACodec.NONE, true)); + + FORMAT_MAP.put(298, new Format(298, "mp4", 720, Format.VCodec.H264, 60, Format.ACodec.NONE, true)); + FORMAT_MAP.put(299, new Format(299, "mp4", 1080, Format.VCodec.H264, 60, Format.ACodec.NONE, true)); // Dash Audio - META_MAP.put(140, new Meta(140, "m4a", Meta.VCodec.NONE, Meta.ACodec.AAC, 128, true)); - META_MAP.put(141, new Meta(141, "m4a", Meta.VCodec.NONE, Meta.ACodec.AAC, 256, true)); + FORMAT_MAP.put(140, new Format(140, "m4a", Format.VCodec.NONE, Format.ACodec.AAC, 128, true)); + FORMAT_MAP.put(141, new Format(141, "m4a", Format.VCodec.NONE, Format.ACodec.AAC, 256, true)); // WEBM Dash Video - META_MAP.put(278, new Meta(278, "webm", 144, Meta.VCodec.VP9, Meta.ACodec.NONE, true)); - META_MAP.put(242, new Meta(242, "webm", 240, Meta.VCodec.VP9, Meta.ACodec.NONE, true)); - META_MAP.put(243, new Meta(243, "webm", 360, Meta.VCodec.VP9, Meta.ACodec.NONE, true)); - META_MAP.put(244, new Meta(244, "webm", 480, Meta.VCodec.VP9, Meta.ACodec.NONE, true)); - META_MAP.put(247, new Meta(247, "webm", 720, Meta.VCodec.VP9, Meta.ACodec.NONE, true)); - META_MAP.put(248, new Meta(248, "webm", 1080, Meta.VCodec.VP9, Meta.ACodec.NONE, true)); - META_MAP.put(271, new Meta(271, "webm", 1440, Meta.VCodec.VP9, Meta.ACodec.NONE, true)); - META_MAP.put(313, new Meta(313, "webm", 2160, Meta.VCodec.VP9, Meta.ACodec.NONE, true)); - - META_MAP.put(302, new Meta(302, "webm", 720, Meta.VCodec.VP9, 60, Meta.ACodec.NONE, true)); - META_MAP.put(308, new Meta(308, "webm", 1440, Meta.VCodec.VP9, 60, Meta.ACodec.NONE, true)); - META_MAP.put(303, new Meta(303, "webm", 1080, Meta.VCodec.VP9, 60, Meta.ACodec.NONE, true)); - META_MAP.put(315, new Meta(315, "webm", 2160, Meta.VCodec.VP9, 60, Meta.ACodec.NONE, true)); + FORMAT_MAP.put(278, new Format(278, "webm", 144, Format.VCodec.VP9, Format.ACodec.NONE, true)); + FORMAT_MAP.put(242, new Format(242, "webm", 240, Format.VCodec.VP9, Format.ACodec.NONE, true)); + FORMAT_MAP.put(243, new Format(243, "webm", 360, Format.VCodec.VP9, Format.ACodec.NONE, true)); + FORMAT_MAP.put(244, new Format(244, "webm", 480, Format.VCodec.VP9, Format.ACodec.NONE, true)); + FORMAT_MAP.put(247, new Format(247, "webm", 720, Format.VCodec.VP9, Format.ACodec.NONE, true)); + FORMAT_MAP.put(248, new Format(248, "webm", 1080, Format.VCodec.VP9, Format.ACodec.NONE, true)); + FORMAT_MAP.put(271, new Format(271, "webm", 1440, Format.VCodec.VP9, Format.ACodec.NONE, true)); + FORMAT_MAP.put(313, new Format(313, "webm", 2160, Format.VCodec.VP9, Format.ACodec.NONE, true)); + + FORMAT_MAP.put(302, new Format(302, "webm", 720, Format.VCodec.VP9, 60, Format.ACodec.NONE, true)); + FORMAT_MAP.put(308, new Format(308, "webm", 1440, Format.VCodec.VP9, 60, Format.ACodec.NONE, true)); + FORMAT_MAP.put(303, new Format(303, "webm", 1080, Format.VCodec.VP9, 60, Format.ACodec.NONE, true)); + FORMAT_MAP.put(315, new Format(315, "webm", 2160, Format.VCodec.VP9, 60, Format.ACodec.NONE, true)); // WEBM Dash Audio - META_MAP.put(171, new Meta(171, "webm", Meta.VCodec.NONE, Meta.ACodec.VORBIS, 128, true)); + FORMAT_MAP.put(171, new Format(171, "webm", Format.VCodec.NONE, Format.ACodec.VORBIS, 128, true)); - META_MAP.put(249, new Meta(249, "webm", Meta.VCodec.NONE, Meta.ACodec.OPUS, 48, true)); - META_MAP.put(250, new Meta(250, "webm", Meta.VCodec.NONE, Meta.ACodec.OPUS, 64, true)); - META_MAP.put(251, new Meta(251, "webm", Meta.VCodec.NONE, Meta.ACodec.OPUS, 160, true)); + FORMAT_MAP.put(249, new Format(249, "webm", Format.VCodec.NONE, Format.ACodec.OPUS, 48, true)); + FORMAT_MAP.put(250, new Format(250, "webm", Format.VCodec.NONE, Format.ACodec.OPUS, 64, true)); + FORMAT_MAP.put(251, new Format(251, "webm", Format.VCodec.NONE, Format.ACodec.OPUS, 160, true)); } - public YouTubeUriExtractor(Context con) { + public YouTubeExtractor(Context con) { context = con; } @Override protected void onPostExecute(SparseArray ytFiles) { - onUrisAvailable(youtubeID, videoTitle, ytFiles); + onExtractionComplete(ytFiles, videoMeta); } - public abstract void onUrisAvailable(String videoId, String videoTitle, SparseArray ytFiles); + + /** + * Start the extraction. + * + * @param youtubeLink the youtube page link or video id + * @param parseDashManifest true if the dash manifest should be downloaded and parsed + * @param includeWebM true if WebM streams should be extracted + */ + public void extract(String youtubeLink, boolean parseDashManifest, boolean includeWebM) { + this.parseDashManifest = parseDashManifest; + this.includeWebM = includeWebM; + this.execute(youtubeLink); + } + + public abstract void onExtractionComplete(SparseArray ytFiles, VideoMeta videoMeta); @Override protected SparseArray doInBackground(String... params) { - youtubeID = null; + videoID = null; String ytUrl = params[0]; if (ytUrl == null) { return null; } Matcher mat = patYouTubePageLink.matcher(ytUrl); if (mat.find()) { - youtubeID = mat.group(3); + videoID = mat.group(3); } else { mat = patYouTubeShortLink.matcher(ytUrl); if (mat.find()) { - youtubeID = mat.group(3); + videoID = mat.group(3); } else if (ytUrl.matches("\\p{Graph}+?")) { - youtubeID = ytUrl; + videoID = ytUrl; } } - if (youtubeID != null) { + if (videoID != null) { try { return getStreamUrls(); } catch (Exception e) { @@ -168,8 +191,8 @@ protected SparseArray doInBackground(String... params) { private SparseArray getStreamUrls() throws IOException, InterruptedException { String ytInfoUrl = (useHttp) ? "http://" : "https://"; - ytInfoUrl += "www.youtube.com/get_video_info?video_id=" + youtubeID + "&eurl=" - + URLEncoder.encode("https://youtube.googleapis.com/v/" + youtubeID, "UTF-8"); + ytInfoUrl += "www.youtube.com/get_video_info?video_id=" + videoID + "&eurl=" + + URLEncoder.encode("https://youtube.googleapis.com/v/" + videoID, "UTF-8"); String dashMpdUrl = null; String streamMap = null; @@ -182,17 +205,17 @@ private SparseArray getStreamUrls() throws IOException, InterruptedExcep streamMap = reader.readLine(); } finally { - if (reader != null) { + if (reader != null) reader.close(); - } urlConnection.disconnect(); } - Pattern patTitle; Matcher mat; String curJsFileName = null; String[] streams; SparseArray encSignatures = null; + parseVideoMeta(streamMap); + // Some videos are using a ciphered signature we need to get the // deciphering js-file from the youtubepage. if (streamMap == null || !streamMap.contains("use_cipher_signature=False")) { @@ -201,7 +224,7 @@ private SparseArray getStreamUrls() throws IOException, InterruptedExcep && (decipherJsFileName == null || decipherFunctions == null || decipherFunctionName == null)) { readDecipherFunctFromCache(); } - getUrl = new URL("https://youtube.com/watch?v=" + youtubeID); + getUrl = new URL("https://youtube.com/watch?v=" + videoID); urlConnection = (HttpURLConnection) getUrl.openConnection(); urlConnection.setRequestProperty("User-Agent", USER_AGENT); try { @@ -215,19 +238,12 @@ private SparseArray getStreamUrls() throws IOException, InterruptedExcep } } } finally { - if (reader != null) { + if (reader != null) reader.close(); - } urlConnection.disconnect(); } encSignatures = new SparseArray<>(); - patTitle = Pattern.compile("\"title\":( |)\"(.*?)(? getStreamUrls() throws IOException, InterruptedExcep dashMpdUrl = URLDecoder.decode(mat.group(1), "UTF-8"); } } - patTitle = Pattern.compile("title=(.*?)(&|\\z)"); - mat = patTitle.matcher(streamMap); - if (mat.find()) { - videoTitle = URLDecoder.decode(mat.group(1), "UTF-8"); - } streamMap = URLDecoder.decode(streamMap, "UTF-8"); } @@ -281,11 +292,11 @@ private SparseArray getStreamUrls() throws IOException, InterruptedExcep itag = Integer.parseInt(mat.group(1)); if (LOGGING) Log.d(LOG_TAG, "Itag found:" + itag); - if (META_MAP.get(itag) == null) { + if (FORMAT_MAP.get(itag) == null) { if (LOGGING) Log.d(LOG_TAG, "Itag not in list:" + itag); continue; - } else if (!includeWebM && META_MAP.get(itag).getExt().equals("webm")) { + } else if (!includeWebM && FORMAT_MAP.get(itag).getExt().equals("webm")) { continue; } } else { @@ -305,9 +316,9 @@ private SparseArray getStreamUrls() throws IOException, InterruptedExcep } if (url != null) { - Meta meta = META_MAP.get(itag); + Format format = FORMAT_MAP.get(itag); String finalUrl = URLDecoder.decode(url, "UTF-8"); - YtFile newVideo = new YtFile(meta, finalUrl); + YtFile newVideo = new YtFile(format, finalUrl); ytFiles.put(itag, newVideo); } } @@ -337,7 +348,7 @@ private SparseArray getStreamUrls() throws IOException, InterruptedExcep } else { String url = ytFiles.get(key).getUrl(); url += "&signature=" + sigs[i]; - YtFile newFile = new YtFile(META_MAP.get(key), url); + YtFile newFile = new YtFile(FORMAT_MAP.get(key), url); ytFiles.put(key, newFile); } } @@ -347,7 +358,7 @@ private SparseArray getStreamUrls() throws IOException, InterruptedExcep if (parseDashManifest && dashMpdUrl != null) { for (int i = 0; i < DASH_PARSE_RETRIES; i++) { try { - // It sometimes failes to connect for no apparent reason. We just retry. + // It sometimes fails to connect for no apparent reason. We just retry. parseDashManifest(dashMpdUrl, ytFiles); break; } catch (IOException io) { @@ -498,9 +509,8 @@ private void parseDashManifest(String dashMpdUrl, SparseArray ytFiles) t dashManifest = reader.readLine(); } finally { - if (reader != null) { + if (reader != null) reader.close(); - } urlConnection.disconnect(); } if (dashManifest == null) @@ -512,9 +522,9 @@ private void parseDashManifest(String dashMpdUrl, SparseArray ytFiles) t Matcher mat2 = patItag.matcher(url); if (mat2.find()) { itag = Integer.parseInt(mat2.group(1)); - if (META_MAP.get(itag) == null) + if (FORMAT_MAP.get(itag) == null) continue; - if (!includeWebM && META_MAP.get(itag).getExt().equals("webm")) + if (!includeWebM && FORMAT_MAP.get(itag).getExt().equals("webm")) continue; } else { continue; @@ -522,12 +532,39 @@ private void parseDashManifest(String dashMpdUrl, SparseArray ytFiles) t url = url.replace("&", "&").replace(",", "%2C"). replace("mime=audio/", "mime=audio%2F"). replace("mime=video/", "mime=video%2F"); - YtFile yf = new YtFile(META_MAP.get(itag), url); + YtFile yf = new YtFile(FORMAT_MAP.get(itag), url); ytFiles.append(itag, yf); } } + private void parseVideoMeta(String getVideoInfo) throws UnsupportedEncodingException { + String title = null, author = null, channelId = null; + long viewCount = 0, length = 0; + Matcher mat = patTitle.matcher(getVideoInfo); + if (mat.find()) { + title = URLDecoder.decode(mat.group(1), "UTF-8"); + } + mat = patAuthor.matcher(getVideoInfo); + if (mat.find()) { + author = URLDecoder.decode(mat.group(1), "UTF-8"); + } + mat = patChannelId.matcher(getVideoInfo); + if (mat.find()) { + channelId = mat.group(1); + } + mat = patLength.matcher(getVideoInfo); + if (mat.find()) { + length = Long.parseLong(mat.group(1)); + } + mat = patViewCount.matcher(getVideoInfo); + if (mat.find()) { + viewCount = Long.parseLong(mat.group(1)); + } + videoMeta = new VideoMeta(videoID, title, author, channelId, length, viewCount); + + } + private void readDecipherFunctFromCache() { if (context != null) { File cacheFile = new File(context.getCacheDir().getAbsolutePath() + "/" + CACHE_FILE_NAME); @@ -608,14 +645,17 @@ private void decipherViaWebView(final SparseArray encSignatures) { if (context == null) { return; } + final StringBuilder stb = new StringBuilder(decipherFunctions + " function decipher("); stb.append("){return "); for (int i = 0; i < encSignatures.size(); i++) { int key = encSignatures.keyAt(i); if (i < encSignatures.size() - 1) - stb.append(decipherFunctionName).append("('").append(encSignatures.get(key)).append("')+\"\\n\"+"); + stb.append(decipherFunctionName).append("('").append(encSignatures.get(key)). + append("')+\"\\n\"+"); else - stb.append(decipherFunctionName).append("('").append(encSignatures.get(key)).append("')"); + stb.append(decipherFunctionName).append("('").append(encSignatures.get(key)). + append("')"); } stb.append("};decipher();"); diff --git a/youtubeExtractor/src/main/java/at/huber/youtubeExtractor/YtFile.java b/youtubeExtractor/src/main/java/at/huber/youtubeExtractor/YtFile.java index 4a849af..b8f8f9f 100644 --- a/youtubeExtractor/src/main/java/at/huber/youtubeExtractor/YtFile.java +++ b/youtubeExtractor/src/main/java/at/huber/youtubeExtractor/YtFile.java @@ -2,11 +2,11 @@ public class YtFile { - private Meta meta; + private Format format; private String url = ""; - YtFile(Meta meta, String url) { - this.meta = meta; + YtFile(Format format, String url) { + this.format = format; this.url = url; } @@ -18,9 +18,9 @@ public String getUrl() { } /** - * Meta data for the specific file. + * Format data for the specific file. */ - public Meta getMeta() { - return meta; + public Format getFormat() { + return format; } } From 5f69586f4791055c67eeb03006f3a0580ec93199 Mon Sep 17 00:00:00 2001 From: HaarigerHarald Date: Fri, 1 Apr 2016 18:17:01 +0200 Subject: [PATCH 2/4] Re-establish old api as deprecated --- .../youtubeExtractor/YouTubeUriExtractor.java | 19 +++++++++++++++++++ .../at/huber/youtubeExtractor/YtFile.java | 8 ++++++++ 2 files changed, 27 insertions(+) create mode 100644 youtubeExtractor/src/main/java/at/huber/youtubeExtractor/YouTubeUriExtractor.java diff --git a/youtubeExtractor/src/main/java/at/huber/youtubeExtractor/YouTubeUriExtractor.java b/youtubeExtractor/src/main/java/at/huber/youtubeExtractor/YouTubeUriExtractor.java new file mode 100644 index 0000000..06c032a --- /dev/null +++ b/youtubeExtractor/src/main/java/at/huber/youtubeExtractor/YouTubeUriExtractor.java @@ -0,0 +1,19 @@ +package at.huber.youtubeExtractor; + +import android.content.Context; +import android.util.SparseArray; + +@Deprecated +public abstract class YouTubeUriExtractor extends YouTubeExtractor { + + public YouTubeUriExtractor(Context con) { + super(con); + } + + @Override + public void onExtractionComplete(SparseArray ytFiles, VideoMeta videoMeta) { + onUrisAvailable(videoMeta.getVideoId(), videoMeta.getTitle(), ytFiles); + } + + public abstract void onUrisAvailable(String videoId, String videoTitle, SparseArray ytFiles); +} diff --git a/youtubeExtractor/src/main/java/at/huber/youtubeExtractor/YtFile.java b/youtubeExtractor/src/main/java/at/huber/youtubeExtractor/YtFile.java index b8f8f9f..6b27af4 100644 --- a/youtubeExtractor/src/main/java/at/huber/youtubeExtractor/YtFile.java +++ b/youtubeExtractor/src/main/java/at/huber/youtubeExtractor/YtFile.java @@ -23,4 +23,12 @@ public String getUrl() { public Format getFormat() { return format; } + + /** + * Format data for the specific file. + */ + @Deprecated + public Format getMeta(){ + return format; + } } From acd498c41cffcd075b0d54255d96a0e8ba53e5c1 Mon Sep 17 00:00:00 2001 From: HaarigerHarald Date: Sun, 17 Apr 2016 18:23:13 +0200 Subject: [PATCH 3/4] Make onExtractionComplete protected --- .../main/java/at/huber/youtubeExtractor/YouTubeExtractor.java | 2 +- .../java/at/huber/youtubeExtractor/YouTubeUriExtractor.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/youtubeExtractor/src/main/java/at/huber/youtubeExtractor/YouTubeExtractor.java b/youtubeExtractor/src/main/java/at/huber/youtubeExtractor/YouTubeExtractor.java index 8e44c67..b447f30 100644 --- a/youtubeExtractor/src/main/java/at/huber/youtubeExtractor/YouTubeExtractor.java +++ b/youtubeExtractor/src/main/java/at/huber/youtubeExtractor/YouTubeExtractor.java @@ -156,7 +156,7 @@ public void extract(String youtubeLink, boolean parseDashManifest, boolean inclu this.execute(youtubeLink); } - public abstract void onExtractionComplete(SparseArray ytFiles, VideoMeta videoMeta); + protected abstract void onExtractionComplete(SparseArray ytFiles, VideoMeta videoMeta); @Override protected SparseArray doInBackground(String... params) { diff --git a/youtubeExtractor/src/main/java/at/huber/youtubeExtractor/YouTubeUriExtractor.java b/youtubeExtractor/src/main/java/at/huber/youtubeExtractor/YouTubeUriExtractor.java index 06c032a..16f9b21 100644 --- a/youtubeExtractor/src/main/java/at/huber/youtubeExtractor/YouTubeUriExtractor.java +++ b/youtubeExtractor/src/main/java/at/huber/youtubeExtractor/YouTubeUriExtractor.java @@ -11,7 +11,7 @@ public YouTubeUriExtractor(Context con) { } @Override - public void onExtractionComplete(SparseArray ytFiles, VideoMeta videoMeta) { + protected void onExtractionComplete(SparseArray ytFiles, VideoMeta videoMeta) { onUrisAvailable(videoMeta.getVideoId(), videoMeta.getTitle(), ytFiles); } From ab01318d1b02d1cd4cf5d69f5d875c6fb2c77fe3 Mon Sep 17 00:00:00 2001 From: HaarigerHarald Date: Thu, 9 Jun 2016 15:54:47 +0200 Subject: [PATCH 4/4] Add support for YouTube Live streams --- README.md | 1 - build.gradle | 2 +- .../huber/youtubeExtractor/ExtractorCase.java | 14 +++-- .../at/huber/youtubeExtractor/Format.java | 20 +++++++ .../at/huber/youtubeExtractor/VideoMeta.java | 11 +++- .../youtubeExtractor/YouTubeExtractor.java | 58 ++++++++++++++++++- 6 files changed, 98 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index 65f385e..50dd78b 100644 --- a/README.md +++ b/README.md @@ -73,7 +73,6 @@ Not signature enciphered Videos may work on lower Android versions (untested). Those videos aren't working: * Everything private (private videos, bought movies, ...) -* Live streams * Unavailable in your country * RTMPE urls (very rare) diff --git a/build.gradle b/build.gradle index 6300e4c..a412ff0 100644 --- a/build.gradle +++ b/build.gradle @@ -5,7 +5,7 @@ buildscript { jcenter() } dependencies { - classpath 'com.android.tools.build:gradle:2.1.0' + classpath 'com.android.tools.build:gradle:2.1.2' // NOTE: Do not place your application dependencies here; they belong // in the individual module build.gradle files diff --git a/youtubeExtractor/src/androidTest/java/at/huber/youtubeExtractor/ExtractorCase.java b/youtubeExtractor/src/androidTest/java/at/huber/youtubeExtractor/ExtractorCase.java index aeb6394..b690a8a 100644 --- a/youtubeExtractor/src/androidTest/java/at/huber/youtubeExtractor/ExtractorCase.java +++ b/youtubeExtractor/src/androidTest/java/at/huber/youtubeExtractor/ExtractorCase.java @@ -18,7 +18,7 @@ public class ExtractorCase extends InstrumentationTestCase { public void testUsualVideo() throws Throwable { VideoMeta expMeta = new VideoMeta("YE7VzlLtp-4", "Big Buck Bunny", "Blender Foundation", - "UCSMOQeBJ2RAnuFungnQOxLg", 597, 0); + "UCSMOQeBJ2RAnuFungnQOxLg", 597, 0, false); extractorTest("http://youtube.com/watch?v=YE7VzlLtp-4", expMeta); extractorTestDashManifest("http://youtube.com/watch?v=YE7VzlLtp-4"); } @@ -26,18 +26,24 @@ public void testUsualVideo() throws Throwable { public void testEncipheredVideo() throws Throwable { VideoMeta expMeta = new VideoMeta("e8X3ACToii0", "Rise Against - Savior", "RiseAgainstVEVO", - "UChMKB2AHNpeuWhalpRYhUaw", 244, 0); + "UChMKB2AHNpeuWhalpRYhUaw", 244, 0, false); extractorTest("https://www.youtube.com/watch?v=e8X3ACToii0", expMeta); - extractorTestDashManifest("https://www.youtube.com/watch?v=e8X3ACToii0"); } public void testAgeRestrictVideo() throws Throwable { VideoMeta expMeta = new VideoMeta("61Ev-YvBw2c", "Test video for age-restriction", - "jpdemoA", "UC95NqtFsDZKlmzOJmZi_g6Q", 14, 0); + "jpdemoA", "UC95NqtFsDZKlmzOJmZi_g6Q", 14, 0, false); extractorTest("http://www.youtube.com/watch?v=61Ev-YvBw2c", expMeta); extractorTestDashManifest("http://www.youtube.com/watch?v=61Ev-YvBw2c"); } + public void testLiveStream() throws Throwable { + VideoMeta expMeta = new VideoMeta("njCDZWTI-xg", "NASA Video : Earth From Space Real Footage - Video From The International Space Station ISS", + "Space Videos", "UCakgsb0w7QB0VHdnCc-OVEA", 1800, 0, true); + extractorTest("http://www.youtube.com/watch?v=njCDZWTI-xg", expMeta); + } + + private void extractorTestDashManifest(final String youtubeLink) throws Throwable { final CountDownLatch signal = new CountDownLatch(1); diff --git a/youtubeExtractor/src/main/java/at/huber/youtubeExtractor/Format.java b/youtubeExtractor/src/main/java/at/huber/youtubeExtractor/Format.java index 3654cdd..7d1021a 100644 --- a/youtubeExtractor/src/main/java/at/huber/youtubeExtractor/Format.java +++ b/youtubeExtractor/src/main/java/at/huber/youtubeExtractor/Format.java @@ -18,6 +18,7 @@ public enum ACodec { private ACodec aCodec; private int audioBitrate; private boolean isDashContainer; + private boolean isHlsContent; Format(int itag, String ext, int height, VCodec vCodec, ACodec aCodec, boolean isDashContainer) { this.itag = itag; @@ -26,6 +27,7 @@ public enum ACodec { this.fps = 30; this.audioBitrate = -1; this.isDashContainer = isDashContainer; + this.isHlsContent = false; } Format(int itag, String ext, VCodec vCodec, ACodec aCodec, int audioBitrate, boolean isDashContainer) { @@ -35,6 +37,7 @@ public enum ACodec { this.fps = 30; this.audioBitrate = audioBitrate; this.isDashContainer = isDashContainer; + this.isHlsContent = false; } Format(int itag, String ext, int height, VCodec vCodec, ACodec aCodec, int audioBitrate, @@ -45,6 +48,18 @@ public enum ACodec { this.fps = 30; this.audioBitrate = audioBitrate; this.isDashContainer = isDashContainer; + this.isHlsContent = false; + } + + Format(int itag, String ext, int height, VCodec vCodec, ACodec aCodec, int audioBitrate, + boolean isDashContainer, boolean isHlsContent) { + this.itag = itag; + this.ext = ext; + this.height = height; + this.fps = 30; + this.audioBitrate = audioBitrate; + this.isDashContainer = isDashContainer; + this.isHlsContent = isHlsContent; } Format(int itag, String ext, int height, VCodec vCodec, int fps, ACodec aCodec, boolean isDashContainer) { @@ -54,6 +69,7 @@ public enum ACodec { this.audioBitrate = -1; this.fps = fps; this.isDashContainer = isDashContainer; + this.isHlsContent = false; } /** @@ -96,6 +112,10 @@ public VCodec getVideoCodec() { return vCodec; } + public boolean isHlsContent() { + return isHlsContent; + } + /** * The pixel height of the video stream or -1 for audio files. */ diff --git a/youtubeExtractor/src/main/java/at/huber/youtubeExtractor/VideoMeta.java b/youtubeExtractor/src/main/java/at/huber/youtubeExtractor/VideoMeta.java index 5627ce2..3fa4707 100644 --- a/youtubeExtractor/src/main/java/at/huber/youtubeExtractor/VideoMeta.java +++ b/youtubeExtractor/src/main/java/at/huber/youtubeExtractor/VideoMeta.java @@ -13,15 +13,20 @@ public class VideoMeta { private long videoLength; private long viewCount; - protected VideoMeta(String videoId, String title, String author, String channelId, long videoLength, long viewCount) { + private boolean isLiveStream; + + protected VideoMeta(String videoId, String title, String author, String channelId, long videoLength, long viewCount, boolean isLiveStream) { this.videoId = videoId; this.title = title; this.author = author; this.channelId = channelId; this.videoLength = videoLength; this.viewCount = viewCount; + this.isLiveStream = isLiveStream; } + + // 120 x 90 public String getThumbUrl() { return IMAGE_BASE_URL + videoId + "/default.jpg"; @@ -63,6 +68,10 @@ public String getChannelId() { return channelId; } + public boolean isLiveStream() { + return isLiveStream; + } + /** * The video length in seconds. */ diff --git a/youtubeExtractor/src/main/java/at/huber/youtubeExtractor/YouTubeExtractor.java b/youtubeExtractor/src/main/java/at/huber/youtubeExtractor/YouTubeExtractor.java index b447f30..0f4a401 100644 --- a/youtubeExtractor/src/main/java/at/huber/youtubeExtractor/YouTubeExtractor.java +++ b/youtubeExtractor/src/main/java/at/huber/youtubeExtractor/YouTubeExtractor.java @@ -71,6 +71,9 @@ public abstract class YouTubeExtractor extends AsyncTask getStreamUrls() throws IOException, InterruptedExcep parseVideoMeta(streamMap); + if(videoMeta.isLiveStream()){ + mat = patHlsvp.matcher(streamMap); + if(mat.find()) { + String hlsvp = URLDecoder.decode(mat.group(1), "UTF-8"); + SparseArray ytFiles = new SparseArray<>(); + + getUrl = new URL(hlsvp); + urlConnection = (HttpURLConnection) getUrl.openConnection(); + urlConnection.setRequestProperty("User-Agent", USER_AGENT); + try { + reader = new BufferedReader(new InputStreamReader(urlConnection.getInputStream())); + String line; + while ((line = reader.readLine()) != null) { + if(line.startsWith("https://") || line.startsWith("http://")){ + mat = patHlsItag.matcher(line); + if(mat.find()){ + int itag = Integer.parseInt(mat.group(1)); + YtFile newFile = new YtFile(FORMAT_MAP.get(itag), line); + ytFiles.put(itag, newFile); + } + } + } + } finally { + if (reader != null) + reader.close(); + urlConnection.disconnect(); + } + + if (ytFiles.size() == 0) { + if (LOGGING) + Log.d(LOG_TAG, streamMap); + return null; + } + return ytFiles; + } + return null; + } + + // Some videos are using a ciphered signature we need to get the // deciphering js-file from the youtubepage. if (streamMap == null || !streamMap.contains("use_cipher_signature=False")) { @@ -539,12 +589,18 @@ private void parseDashManifest(String dashMpdUrl, SparseArray ytFiles) t } private void parseVideoMeta(String getVideoInfo) throws UnsupportedEncodingException { + boolean isLiveStream = false; String title = null, author = null, channelId = null; long viewCount = 0, length = 0; Matcher mat = patTitle.matcher(getVideoInfo); if (mat.find()) { title = URLDecoder.decode(mat.group(1), "UTF-8"); } + + mat = patHlsvp.matcher(getVideoInfo); + if(mat.find()) + isLiveStream = true; + mat = patAuthor.matcher(getVideoInfo); if (mat.find()) { author = URLDecoder.decode(mat.group(1), "UTF-8"); @@ -561,7 +617,7 @@ private void parseVideoMeta(String getVideoInfo) throws UnsupportedEncodingExcep if (mat.find()) { viewCount = Long.parseLong(mat.group(1)); } - videoMeta = new VideoMeta(videoID, title, author, channelId, length, viewCount); + videoMeta = new VideoMeta(videoID, title, author, channelId, length, viewCount, isLiveStream); }