Skip to content

Commit b2bbddf

Browse files
gaeqsgaeqs
committed
Improved embedded url extraction.
1 parent 36cbd9e commit b2bbddf

File tree

5 files changed

+1909
-35
lines changed

5 files changed

+1909
-35
lines changed

‎JavaYoutubeDownloader.iml‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
<?xml version="1.0" encoding="UTF-8"?>
2+
<module org.jetbrains.idea.maven.project.MavenProjectsManager.isMavenModule="true" type="JAVA_MODULE" version="4">
3+
<component name="NewModuleRootManager" LANGUAGE_LEVEL="JDK_1_8">
4+
<output url="file://$MODULE_DIR$/target/classes" />
5+
<output-test url="file://$MODULE_DIR$/target/test-classes" />
6+
<content url="file://$MODULE_DIR$">
7+
<sourceFolder url="file://$MODULE_DIR$/src/main/java" isTestSource="false" />
8+
<sourceFolder url="file://$MODULE_DIR$/src/main/test" isTestSource="true" />
9+
<excludeFolder url="file://$MODULE_DIR$/target" />
10+
</content>
11+
<orderEntry type="inheritedJdk" />
12+
<orderEntry type="sourceFolder" forTests="false" />
13+
<orderEntry type="library" scope="TEST" name="Maven: org.junit.jupiter:junit-jupiter:5.6.0-M1" level="project" />
14+
<orderEntry type="library" scope="TEST" name="Maven: org.junit.jupiter:junit-jupiter-api:5.6.0-M1" level="project" />
15+
<orderEntry type="library" scope="TEST" name="Maven: org.apiguardian:apiguardian-api:1.1.0" level="project" />
16+
<orderEntry type="library" scope="TEST" name="Maven: org.opentest4j:opentest4j:1.2.0" level="project" />
17+
<orderEntry type="library" scope="TEST" name="Maven: org.junit.platform:junit-platform-commons:1.6.0-M1" level="project" />
18+
<orderEntry type="library" scope="TEST" name="Maven: org.junit.jupiter:junit-jupiter-params:5.6.0-M1" level="project" />
19+
<orderEntry type="library" scope="TEST" name="Maven: org.junit.jupiter:junit-jupiter-engine:5.6.0-M1" level="project" />
20+
<orderEntry type="library" scope="TEST" name="Maven: org.junit.platform:junit-platform-engine:1.6.0-M1" level="project" />
21+
<orderEntry type="library" name="Maven: org.json:json:20190722" level="project" />
22+
</component>
23+
</module>

‎pom.xml‎

Lines changed: 14 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -18,30 +18,20 @@
1818
<target>8</target>
1919
</configuration>
2020
</plugin>
21-
<plugin>
22-
<groupId>org.apache.maven.plugins</groupId>
23-
<artifactId>maven-javadoc-plugin</artifactId>
24-
<executions>
25-
<execution>
26-
<id>attach-javadocs</id>
27-
<goals>
28-
<goal>jar</goal>
29-
</goals>
30-
</execution>
31-
</executions>
32-
</plugin>
33-
<plugin>
34-
<groupId>org.apache.maven.plugins</groupId>
35-
<artifactId>maven-source-plugin</artifactId>
36-
<executions>
37-
<execution>
38-
<id>attach-sources</id>
39-
<goals>
40-
<goal>jar</goal>
41-
</goals>
42-
</execution>
43-
</executions>
44-
</plugin>
4521
</plugins>
4622
</build>
23+
<dependencies>
24+
<dependency>
25+
<groupId>org.junit.jupiter</groupId>
26+
<artifactId>junit-jupiter</artifactId>
27+
<version>RELEASE</version>
28+
<scope>test</scope>
29+
</dependency>
30+
<dependency>
31+
<groupId>org.json</groupId>
32+
<artifactId>json</artifactId>
33+
<version>20190722</version>
34+
</dependency>
35+
</dependencies>
36+
4737
</project>

‎src/main/java/com/degoos/javayoutubedownloader/decoder/EmbeddedDecoder.java‎

Lines changed: 52 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -6,15 +6,14 @@
66
import com.degoos.javayoutubedownloader.util.EncodedStreamUtils;
77
import com.degoos.javayoutubedownloader.util.HTMLUtils;
88
import com.degoos.javayoutubedownloader.util.IdExtractor;
9+
import org.json.JSONArray;
10+
import org.json.JSONObject;
911

1012
import java.io.IOException;
1113
import java.io.UnsupportedEncodingException;
1214
import java.net.URL;
1315
import java.net.URLDecoder;
14-
import java.util.HashMap;
15-
import java.util.LinkedList;
16-
import java.util.List;
17-
import java.util.Map;
16+
import java.util.*;
1817

1918
/**
2019
* This class represents a decoder that uses the Youtube's embedded API to decode stream options.
@@ -34,6 +33,10 @@ public class EmbeddedDecoder implements Decoder {
3433
public static final String MUXED_STREAM_LIST_PARAMETER = "url_encoded_fmt_stream_map";
3534
public static final String ADAPTIVE_STREAM_LIST_PARAMETER = "adaptive_fmts";
3635

36+
public static final String PLAYER_RESPONSE_LIST_PARAMETER = "player_response";
37+
public static final String STREAMING_DATA_JSON_PARAMETER = "streamingData";
38+
public static final String FORMATS_JSON_PARAMETER = "formats";
39+
3740
private String urlEncoding;
3841
private String getVideoUrl;
3942

@@ -69,15 +72,37 @@ public YoutubeVideo extractVideo(URL url) throws IOException {
6972
Map<String, String> queryData = getQueryMap(query);
7073
checkExceptions(queryData);
7174

72-
String title = decode(queryData.get(TITLE_PARAMETER));
73-
String author = decode(queryData.get(AUTHOR_PARAMETER));
75+
queryData.forEach((key, value) -> System.out.println(key + "=" + value));
76+
77+
String title = queryData.containsKey(TITLE_PARAMETER) ? decode(queryData.get(TITLE_PARAMETER)) : "null";
78+
String author = queryData.containsKey(AUTHOR_PARAMETER) ? decode(queryData.get(AUTHOR_PARAMETER)) : "null";
7479
YoutubeVideo video = new YoutubeVideo(title, author);
7580

76-
String encodedMuxedStreamList = decode(queryData.get(MUXED_STREAM_LIST_PARAMETER));
77-
String encodedAdaptiveStreamList = decode(queryData.get(ADAPTIVE_STREAM_LIST_PARAMETER));
7881
List<EncodedStream> encodedStreams = new LinkedList<>();
79-
EncodedStreamUtils.addEncodedStreams(encodedMuxedStreamList, encodedStreams, urlEncoding);
80-
EncodedStreamUtils.addEncodedStreams(encodedAdaptiveStreamList, encodedStreams, urlEncoding);
82+
83+
//Player response data.
84+
if (queryData.containsKey(PLAYER_RESPONSE_LIST_PARAMETER)) {
85+
JSONObject obj = new JSONObject(decode(queryData.get(PLAYER_RESPONSE_LIST_PARAMETER)));
86+
if (obj.has(STREAMING_DATA_JSON_PARAMETER)) {
87+
obj = obj.getJSONObject(STREAMING_DATA_JSON_PARAMETER);
88+
if (obj.has(FORMATS_JSON_PARAMETER)) {
89+
addJSONStreams(obj.getJSONArray(FORMATS_JSON_PARAMETER), encodedStreams);
90+
}
91+
}
92+
}
93+
94+
//Muxed stream data.
95+
if (queryData.containsKey(MUXED_STREAM_LIST_PARAMETER)) {
96+
String encodedMuxedStreamList = decode(queryData.get(MUXED_STREAM_LIST_PARAMETER));
97+
EncodedStreamUtils.addEncodedStreams(encodedMuxedStreamList, encodedStreams, urlEncoding);
98+
}
99+
100+
//Adaptive stream data.
101+
if (queryData.containsKey(ADAPTIVE_STREAM_LIST_PARAMETER)) {
102+
String encodedAdaptiveStreamList = decode(queryData.get(ADAPTIVE_STREAM_LIST_PARAMETER));
103+
EncodedStreamUtils.addEncodedStreams(encodedAdaptiveStreamList, encodedStreams, urlEncoding);
104+
}
105+
81106
encodedStreams.removeIf(target -> !target.decode(null, true));
82107
encodedStreams.forEach(target -> video.getStreamOptions().add(target.getDecodedStream()));
83108
return video;
@@ -111,9 +136,25 @@ private String checkExceptions(Map<String, String> queryMap) {
111136
private String decode(String string) {
112137
try {
113138
return URLDecoder.decode(string, urlEncoding);
114-
} catch (UnsupportedEncodingException e) {
139+
} catch (UnsupportedEncodingException | NullPointerException e) {
140+
System.err.println("Error while decoding string " + string);
115141
e.printStackTrace();
116142
return string;
117143
}
118144
}
145+
146+
private void addJSONStreams(JSONArray array, Collection<EncodedStream> streams) {
147+
array.forEach(target -> {
148+
try {
149+
if (!(target instanceof JSONObject)) return;
150+
JSONObject obj = (JSONObject) target;
151+
int iTag = obj.getInt("itag");
152+
String url = URLDecoder.decode(obj.getString("url"), urlEncoding);
153+
streams.add(new EncodedStream(iTag, url));
154+
} catch (UnsupportedEncodingException e) {
155+
System.err.println("Error while parsing url.");
156+
e.printStackTrace();
157+
}
158+
});
159+
}
119160
}

‎src/main/test/com/degoos/javayoutubedownloader/JavaYoutubeDownloaderTest.java‎

Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,70 @@
1+
package com.degoos.javayoutubedownloader;
2+
3+
import com.degoos.javayoutubedownloader.decoder.MultipleDecoderMethod;
4+
import com.degoos.javayoutubedownloader.stream.YoutubeVideo;
5+
import org.junit.jupiter.api.Test;
6+
7+
import java.net.MalformedURLException;
8+
9+
import static org.junit.jupiter.api.Assertions.assertFalse;
10+
import static org.junit.jupiter.api.Assertions.assertNotNull;
11+
12+
class JavaYoutubeDownloaderTest {
13+
14+
@Test
15+
public void htmlTest() throws MalformedURLException {
16+
YoutubeVideo video = JavaYoutubeDownloader.decode("https://www.youtube.com/watch?v=JmzMzAbR6D0",
17+
MultipleDecoderMethod.AND, "html");
18+
assertNotNull(video, "Video is null.");
19+
assertFalse(video.getStreamOptions().isEmpty(), "Video options list is empty.");
20+
System.out.println("URLs:");
21+
video.getStreamOptions().forEach(target -> System.out.println(target.getUrl()));
22+
}
23+
24+
@Test
25+
public void embeddedTest() throws MalformedURLException {
26+
YoutubeVideo video = JavaYoutubeDownloader.decode("https://www.youtube.com/watch?v=JmzMzAbR6D0",
27+
MultipleDecoderMethod.AND, "embedded");
28+
assertNotNull(video, "Video is null.");
29+
assertFalse(video.getStreamOptions().isEmpty(), "Video options list is empty.");
30+
video.getStreamOptions().forEach(target -> System.out.println(target.getUrl()));
31+
}
32+
33+
@Test
34+
public void htmlTestOldVideo() throws MalformedURLException {
35+
YoutubeVideo video = JavaYoutubeDownloader.decode("https://www.youtube.com/watch?v=l_jUBScR1RA",
36+
MultipleDecoderMethod.AND, "html");
37+
assertNotNull(video, "Video is null.");
38+
assertFalse(video.getStreamOptions().isEmpty(), "Video options list is empty.");
39+
System.out.println("URLs:");
40+
video.getStreamOptions().forEach(target -> System.out.println(target.getUrl()));
41+
}
42+
43+
@Test
44+
public void embeddedTestOldVideo() throws MalformedURLException {
45+
YoutubeVideo video = JavaYoutubeDownloader.decode("https://www.youtube.com/watch?v=l_jUBScR1RA",
46+
MultipleDecoderMethod.AND, "embedded");
47+
assertNotNull(video, "Video is null.");
48+
assertFalse(video.getStreamOptions().isEmpty(), "Video options list is empty.");
49+
video.getStreamOptions().forEach(target -> System.out.println(target.getUrl()));
50+
}
51+
52+
@Test
53+
public void htmlTestProtected() throws MalformedURLException {
54+
YoutubeVideo video = JavaYoutubeDownloader.decode("https://www.youtube.com/watch?v=Nx-DvH41Tjo",
55+
MultipleDecoderMethod.AND, "html");
56+
assertNotNull(video, "Video is null.");
57+
assertFalse(video.getStreamOptions().isEmpty(), "Video options list is empty.");
58+
video.getStreamOptions().forEach(target -> System.out.println(target.getUrl()));
59+
}
60+
61+
@Test
62+
public void embeddedTestProtected() throws MalformedURLException {
63+
YoutubeVideo video = JavaYoutubeDownloader.decode("https://www.youtube.com/watch?v=Nx-DvH41Tjo",
64+
MultipleDecoderMethod.AND, "embedded");
65+
assertNotNull(video, "Video is null.");
66+
assertFalse(video.getStreamOptions().isEmpty(), "Video options list is empty.");
67+
video.getStreamOptions().forEach(target -> System.out.println(target.getUrl()));
68+
}
69+
70+
}

0 commit comments

Comments
 (0)