Skip to content

Commit 0a56c26

Browse files
phireskyphiresky
committed
working pretty much
1 parent 19b199a commit 0a56c26

File tree

9 files changed

+109
-22
lines changed

9 files changed

+109
-22
lines changed

‎create_db.sh‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -6,5 +6,6 @@ cat create_db.sql | sqlite3 -cmd '.echo on' dist/data/db.sqlite3
66
bytes="$(stat --printf="%s" dist/data/db.sqlite3)"
77
serverChunkSize=$((50 * 1024 * 1024))
88
suffixLength=3
9-
split data/db.sqlite3 --bytes=$serverChunkSize dist/data/db.sqlite3. --suffix-length=$suffixLength --numeric-suffixes
9+
split dist/data/db.sqlite3 --bytes=$serverChunkSize dist/data/db.sqlite3. --suffix-length=$suffixLength --numeric-suffixes
10+
rm dist/data/db.sqlite3
1011
echo '{"requestChunkSize": 4096, "databaseLengthBytes": '$bytes', "serverChunkSize": '$serverChunkSize', "urlPrefix": "db.sqlite3.", "suffixLength": '$suffixLength'}' > dist/data/config.json

‎create_db.sql‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@ pragma page_size = 4096;
22
-- 1024: db.sqlite3 total bytes fetched: 71632 total requests: 48
33
-- 4096: db.sqlite3 total bytes fetched: 98286 total requests: 18
44

5-
attach database 'youtube-metadata-pg4096.sqlite3' as ytm;
5+
attach database 'data/youtube-metadata-pg4096.sqlite3' as ytm;
66

77
CREATE TABLE authors (id integer primary key autoincrement, name text not null unique);
88

‎package.json‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
{
22
"scripts": {
3-
"dev": "webpack serve"
3+
"dev": "webpack serve --mode=development",
4+
"build": "webpack build --mode=production"
45
},
56
"dependencies": {
67
"@types/debounce-promise": "^3.1.3",
@@ -26,6 +27,7 @@
2627
"worker-loader": "^3.0.8"
2728
},
2829
"devDependencies": {
30+
"webpack-bundle-analyzer": "^4.4.1",
2931
"webpack-dev-server": "^3.11.2"
3032
}
3133
}

‎src/UI.tsx‎

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,13 @@ import {
1212
import { action, makeAutoObservable, makeObservable, observable } from "mobx";
1313
import AsyncSelect from "react-select/async";
1414
import debounce from "debounce-promise";
15-
import Plot from "react-plotly.js";
15+
import createPlotlyComponent from "react-plotly.js/factory";
16+
import Plotly from "plotly.js/lib/core";
17+
import { textChangeRangeIsUnchanged } from "typescript";
18+
19+
20+
const Plot = createPlotlyComponent(Plotly);
21+
1622
function formatDuration(sec_num: number) {
1723
const hours = Math.floor(sec_num / 3600);
1824
const minutes = Math.floor((sec_num - hours * 3600) / 60);
@@ -30,7 +36,6 @@ const SponsorPlot: React.FC<{
3036
data: SponsorInfo[];
3137
onHover: (m: SponsorInfo) => void;
3238
}> = observer((p) => {
33-
console.log("RERENTDERING PLOT");
3439
return (
3540
<Plot
3641
style={{ width: "100%", maxWidth: "1200px", margin: "0 auto" }}
@@ -123,6 +128,7 @@ const VideoMetaDisplay: React.FC<{ video: SponsorInfo }> = observer(
123128
export class UI extends React.Component {
124129
worker: SqliteWorker | null = null;
125130
db: Database | null = null;
131+
@observable initState = "Loading...";
126132
@observable
127133
data:
128134
| { state: "noinput" }
@@ -151,12 +157,13 @@ export class UI extends React.Component {
151157
clearInterval(this.interval);
152158
}
153159
async init() {
154-
console.log("INIT");
160+
this.initState = "connectingToDb";
155161
const res = await createDbWorker();
156162
this.db = res.db;
157163
this.worker = res.worker;
158164
const initialAuthor = new URLSearchParams(location.search).get("uploader");
159165
if (initialAuthor) this.setAuthor(initialAuthor);
166+
this.initState = "";
160167
}
161168
async setAuthor(search: string) {
162169
this.searchInput = search;
@@ -192,6 +199,7 @@ export class UI extends React.Component {
192199
setFocussed = (e: SponsorInfo) => (this.focussedVideo = e);
193200

194201
render() {
202+
if(this.initState) return <div>{this.initState}</div>;
195203
return (
196204
<div>
197205
<div>

‎src/db.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ export async function createDbWorker() {
2222
const sqlite = Comlink.wrap<SqliteMod>(new SqliteWorker());
2323

2424
const chunkSize = 4096;
25-
const configUrl = new URL("dist/data/config.json", location.href);
25+
const configUrl = new URL("data/config.json", location.href);
2626
const config: SplitFileConfig = await fetch(configUrl.toString()).then(e => e.json());
2727
const db = await sqlite.SplitFileHttpDatabase({
2828
...config,

‎src/index.tsx‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,9 @@
11
import React from "react";
22
import { render } from "react-dom";
3-
import { UI } from "./UI";
3+
import { UI } from "./UI";import{configure} from "mobx";
4+
5+
configure({
6+
enforceActions: "never",
7+
})
48

59
render(<UI />, document.getElementById("root"));

‎src/lazyFile.ts‎

Lines changed: 12 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@ export type LazyFileConfig = {
1818

1919
// Lazy chunked Uint8Array (implements get and length from Uint8Array). Actual getting is abstracted away for eventual reuse.
2020
class LazyUint8Array {
21-
lengthKnown = false;
21+
serverChecked = false;
2222
chunks: Uint8Array[] = []; // Loaded chunks. Index is the chunk number
2323
totalFetchedBytes = 0;
2424
totalRequests = 0;
@@ -32,9 +32,8 @@ class LazyUint8Array {
3232
constructor(config: LazyFileConfig) {
3333
this._chunkSize = config.requestChunkSize;
3434
this.rangeMapper = config.rangeMapper;
35-
if(config.fileLength) {
35+
if (config.fileLength) {
3636
this._length = config.fileLength;
37-
this.lengthKnown = true;
3837
}
3938
}
4039
get(idx: number) {
@@ -79,7 +78,7 @@ class LazyUint8Array {
7978
throw new Error("doXHR failed!");
8079
return this.chunks[chunkNum];
8180
}
82-
cacheLength() {
81+
checkServer() {
8382
// Find length
8483
var xhr = new XMLHttpRequest();
8584
const url = this.rangeMapper(0, 0).url;
@@ -95,23 +94,25 @@ class LazyUint8Array {
9594
var usesGzip =
9695
(header = xhr.getResponseHeader("Content-Encoding")) && header === "gzip";
9796

98-
if (usesGzip || !datalength || !hasByteServing) {
97+
if (!hasByteServing) throw Error("server does not support byte serving");
98+
99+
if (usesGzip || !datalength) {
99100
throw Error("server uses gzip or doesn't have length");
100101
}
101102

102-
this._length = datalength;
103-
this.lengthKnown = true;
103+
if (!this._length) this._length = datalength;
104+
this.serverChecked = true;
104105
}
105106
get length() {
106-
if (!this.lengthKnown) {
107-
this.cacheLength();
107+
if (!this.serverChecked) {
108+
this.checkServer();
108109
}
109110
return this._length!;
110111
}
111112

112113
get chunkSize() {
113-
if (!this.lengthKnown) {
114-
this.cacheLength();
114+
if (!this.serverChecked) {
115+
this.checkServer();
115116
}
116117
return this._chunkSize!;
117118
}

‎webpack.config.js‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,12 +1,14 @@
11
const path = require("path");
2+
const BundleAnalyzerPlugin = require("webpack-bundle-analyzer")
3+
.BundleAnalyzerPlugin;
24

35
const ts = {
46
loader: "ts-loader",
57
options: { transpileOnly: true },
68
};
79
module.exports = {
810
entry: "./src",
9-
mode: "development",
11+
// mode:,
1012
devtool: "source-map",
1113
module: {
1214
rules: [
@@ -33,12 +35,15 @@ module.exports = {
3335
output: {
3436
filename: "bundle.js",
3537
path: path.resolve(__dirname, "dist"),
38+
assetModuleFilename: "images/[name]-[hash][ext][query]",
3639
},
3740
stats: {
3841
children: true,
3942
},
4043
devServer: {
44+
publicPath: "/dist",
4145
hot: false,
4246
liveReload: false,
4347
},
48+
plugins: [new BundleAnalyzerPlugin()],
4449
};

‎yarn.lock‎

Lines changed: 68 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -180,6 +180,11 @@
180180
parse-rect "^1.2.0"
181181
pick-by-alias "^1.2.0"
182182

183+
"@polka/url@^1.0.0-next.9":
184+
version "1.0.0-next.12"
185+
resolved "https://registry.yarnpkg.com/@polka/url/-/url-1.0.0-next.12.tgz#431ec342a7195622f86688bbda82e3166ce8cb28"
186+
integrity sha512-6RglhutqrGFMO1MNUXp95RBuYIuc8wTnMAV5MUhLmjTOy78ncwOw7RgeQ/HeymkKXRhZd0s2DNrM1rL7unk3MQ==
187+
183188
"@turf/area@^6.0.1":
184189
version "6.3.0"
185190
resolved "https://registry.yarnpkg.com/@turf/area/-/area-6.3.0.tgz#cdd02f8ca51da2889dfc90a3c9e8fef5d1e04dca"
@@ -516,6 +521,11 @@ accepts@~1.3.4, accepts@~1.3.5, accepts@~1.3.7:
516521
mime-types "~2.1.24"
517522
negotiator "0.6.2"
518523

524+
acorn-walk@^8.0.0:
525+
version "8.0.2"
526+
resolved "https://registry.yarnpkg.com/acorn-walk/-/acorn-walk-8.0.2.tgz#d4632bfc63fd93d0f15fd05ea0e984ffd3f5a8c3"
527+
integrity sha512-+bpA9MJsHdZ4bgfDcpk0ozQyhhVct7rzOmO0s1IIr0AGGgKBljss8n2zp11rRP2wid5VGeh04CgeKzgat5/25A==
528+
519529
acorn@^7.1.1:
520530
version "7.4.1"
521531
resolved "https://registry.yarnpkg.com/acorn/-/acorn-7.4.1.tgz#feaed255973d2e77555b83dbc08851a6c63520fa"
@@ -1218,6 +1228,11 @@ commander@2, commander@^2.15.1, commander@^2.20.0:
12181228
resolved "https://registry.yarnpkg.com/commander/-/commander-2.20.3.tgz#fd485e84c03eb4881c20722ba48035e8531aeb33"
12191229
integrity sha512-GpVkmM8vF2vQUkj2LvZmD35JxeJOLCwJ9cUkugyk2nuhbv3+mJvpLYYt+0+USMxE+oj+ey/lJEnhZw75x/OMcQ==
12201230

1231+
commander@^6.2.0:
1232+
version "6.2.1"
1233+
resolved "https://registry.yarnpkg.com/commander/-/commander-6.2.1.tgz#0792eb682dfbc325999bb2b84fddddba110ac73c"
1234+
integrity sha512-U7VdrJFnJgo4xjrHpTzu0yrHPGImdsmD95ZlgYSEajAn2JKzDhDTPG9kBTefmObL2w/ngeZnilk+OV9CG3d7UA==
1235+
12211236
commander@^7.0.0:
12221237
version "7.2.0"
12231238
resolved "https://registry.yarnpkg.com/commander/-/commander-7.2.0.tgz#a36cb57d0b501ce108e4d20559a150a391d97ab7"
@@ -1742,6 +1757,11 @@ dup@^1.0.0:
17421757
resolved "https://registry.yarnpkg.com/dup/-/dup-1.0.0.tgz#51fc5ac685f8196469df0b905e934b20af5b4029"
17431758
integrity sha1-UfxaxoX4GWRp3wuQXpNLIK9bQCk=
17441759

1760+
duplexer@^0.1.2:
1761+
version "0.1.2"
1762+
resolved "https://registry.yarnpkg.com/duplexer/-/duplexer-0.1.2.tgz#3abe43aef3835f8ae077d136ddce0f276b0400e6"
1763+
integrity sha512-jtD6YG370ZCIi/9GTaJKQxWTZD045+4R4hTk/x1UyoqadyJ9x9CgSi1RlVDQF8U2sxLLSnFkCaMihqljHIWgMg==
1764+
17451765
duplexify@^3.4.5:
17461766
version "3.7.1"
17471767
resolved "https://registry.yarnpkg.com/duplexify/-/duplexify-3.7.1.tgz#2a4df5317f6ccfd91f86d6fd25d8d8a103b88309"
@@ -2899,6 +2919,13 @@ grid-index@^1.1.0:
28992919
resolved "https://registry.yarnpkg.com/grid-index/-/grid-index-1.1.0.tgz#97f8221edec1026c8377b86446a7c71e79522ea7"
29002920
integrity sha512-HZRwumpOGUrHyxO5bqKZL0B0GlUpwtCAzZ42sgxUPniu33R1LSFH5yrIcBCHjkctCAh3mtWKcKd9J4vDDdeVHA==
29012921

2922+
gzip-size@^6.0.0:
2923+
version "6.0.0"
2924+
resolved "https://registry.yarnpkg.com/gzip-size/-/gzip-size-6.0.0.tgz#065367fd50c239c0671cbcbad5be3e2eeb10e462"
2925+
integrity sha512-ax7ZYomf6jqPTQ4+XCpUGyXKHk5WweS+e05MBO4/y3WJ5RkmPXNKvX+bx1behVILVwr6JSQvZAku021CHPXG3Q==
2926+
dependencies:
2927+
duplexer "^0.1.2"
2928+
29022929
handle-thing@^2.0.0:
29032930
version "2.0.1"
29042931
resolved "https://registry.yarnpkg.com/handle-thing/-/handle-thing-2.0.1.tgz#857f79ce359580c340d43081cc648970d0bb234e"
@@ -3570,7 +3597,7 @@ locate-path@^5.0.0:
35703597
dependencies:
35713598
p-locate "^4.1.0"
35723599

3573-
lodash@^4.17.11, lodash@^4.17.14:
3600+
lodash@^4.17.11, lodash@^4.17.14, lodash@^4.17.20:
35743601
version "4.17.21"
35753602
resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.21.tgz#679591c564c3bffaae8454cf0b3df370c3d6911c"
35763603
integrity sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==
@@ -3780,7 +3807,7 @@ [email protected]:
37803807
resolved "https://registry.yarnpkg.com/mime/-/mime-1.6.0.tgz#32cd9e5c64553bd58d19a568af452acff04981b1"
37813808
integrity sha512-x0Vn8spI+wuJ1O6S7gnbaQg8Pxh4NNHb7KSINmEWKiPE4RKOplvijn+NkmYmmRgP68mc70j2EbeTFRsrswaQeg==
37823809

3783-
mime@^2.4.4:
3810+
mime@^2.3.1, mime@^2.4.4:
37843811
version "2.5.2"
37853812
resolved "https://registry.yarnpkg.com/mime/-/mime-2.5.2.tgz#6e3dc6cc2b9510643830e5f19d5cb753da5eeabe"
37863813
integrity sha512-tqkh47FzKeCPD2PUiPB6pkbMzsCasjxAfC62/Wap5qrUWcb+sFasXUC5I3gYM5iBM8v/Qpn4UK0x+j0iHyFPDg==
@@ -4169,6 +4196,11 @@ onetime@^5.1.2:
41694196
dependencies:
41704197
mimic-fn "^2.1.0"
41714198

4199+
opener@^1.5.2:
4200+
version "1.5.2"
4201+
resolved "https://registry.yarnpkg.com/opener/-/opener-1.5.2.tgz#5d37e1f35077b9dcac4301372271afdeb2a13598"
4202+
integrity sha512-ur5UIdyw5Y7yEj9wLzhqXiy6GZ3Mwx0yGI+5sMn2r0N0v3cKJvUmFH5yPP+WXh9e0xfyzyJX95D8l088DNFj7A==
4203+
41724204
opn@^5.5.0:
41734205
version "5.5.0"
41744206
resolved "https://registry.yarnpkg.com/opn/-/opn-5.5.0.tgz#fc7164fab56d235904c51c3b27da6758ca3b9bfc"
@@ -5334,6 +5366,15 @@ simplify-planar-graph@^2.0.1:
53345366
robust-orientation "^1.0.1"
53355367
simplicial-complex "^0.3.3"
53365368

5369+
sirv@^1.0.7:
5370+
version "1.0.11"
5371+
resolved "https://registry.yarnpkg.com/sirv/-/sirv-1.0.11.tgz#81c19a29202048507d6ec0d8ba8910fda52eb5a4"
5372+
integrity sha512-SR36i3/LSWja7AJNRBz4fF/Xjpn7lQFI30tZ434dIy+bitLYSP+ZEenHg36i23V2SGEz+kqjksg0uOGZ5LPiqg==
5373+
dependencies:
5374+
"@polka/url" "^1.0.0-next.9"
5375+
mime "^2.3.1"
5376+
totalist "^1.0.0"
5377+
53375378
slab-decomposition@^1.0.1:
53385379
version "1.0.2"
53395380
resolved "https://registry.yarnpkg.com/slab-decomposition/-/slab-decomposition-1.0.2.tgz#1ded56754d408b10739f145103dfc61807f65134"
@@ -5801,6 +5842,11 @@ topojson-client@^3.1.0:
58015842
dependencies:
58025843
commander "2"
58035844

5845+
totalist@^1.0.0:
5846+
version "1.1.0"
5847+
resolved "https://registry.yarnpkg.com/totalist/-/totalist-1.1.0.tgz#a4d65a3e546517701e3e5c37a47a70ac97fe56df"
5848+
integrity sha512-gduQwd1rOdDMGxFG1gEvhV88Oirdo2p+KjoYFU7k2g+i7n6AFFbDQ5kMPUsW0pNbfQsB/cwXvT1i4Bue0s9g5g==
5849+
58045850
triangulate-hypercube@^1.0.0:
58055851
version "1.0.1"
58065852
resolved "https://registry.yarnpkg.com/triangulate-hypercube/-/triangulate-hypercube-1.0.1.tgz#d8071db2ebfcfd51f308d0bcf2a5c48a5b36d137"
@@ -6182,6 +6228,21 @@ webgl-context@^2.2.0:
61826228
dependencies:
61836229
get-canvas-context "^1.0.1"
61846230

6231+
webpack-bundle-analyzer@^4.4.1:
6232+
version "4.4.1"
6233+
resolved "https://registry.yarnpkg.com/webpack-bundle-analyzer/-/webpack-bundle-analyzer-4.4.1.tgz#c71fb2eaffc10a4754d7303b224adb2342069da1"
6234+
integrity sha512-j5m7WgytCkiVBoOGavzNokBOqxe6Mma13X1asfVYtKWM3wxBiRRu1u1iG0Iol5+qp9WgyhkMmBAcvjEfJ2bdDw==
6235+
dependencies:
6236+
acorn "^8.0.4"
6237+
acorn-walk "^8.0.0"
6238+
chalk "^4.1.0"
6239+
commander "^6.2.0"
6240+
gzip-size "^6.0.0"
6241+
lodash "^4.17.20"
6242+
opener "^1.5.2"
6243+
sirv "^1.0.7"
6244+
ws "^7.3.1"
6245+
61856246
webpack-cli@^4.6.0:
61866247
version "4.6.0"
61876248
resolved "https://registry.yarnpkg.com/webpack-cli/-/webpack-cli-4.6.0.tgz#27ae86bfaec0cf393fcfd58abdc5a229ad32fd16"
@@ -6384,6 +6445,11 @@ ws@^6.2.1:
63846445
dependencies:
63856446
async-limiter "~1.0.0"
63866447

6448+
ws@^7.3.1:
6449+
version "7.4.4"
6450+
resolved "https://registry.yarnpkg.com/ws/-/ws-7.4.4.tgz#383bc9742cb202292c9077ceab6f6047b17f2d59"
6451+
integrity sha512-Qm8k8ojNQIMx7S+Zp8u/uHOx7Qazv3Yv4q68MiWWWOJhiwG5W3x7iqmRtJo8xxrciZUY4vRxUTJCKuRnF28ZZw==
6452+
63876453
"xtend@>=4.0.0 <4.1.0-0", xtend@^4.0.0, xtend@~4.0.1:
63886454
version "4.0.2"
63896455
resolved "https://registry.yarnpkg.com/xtend/-/xtend-4.0.2.tgz#bb72779f5fa465186b1f438f674fa347fdb5db54"

0 commit comments

Comments
 (0)