First working version

2026-03-05 13:20:27 -05:00 · 2021-07-20 08:39:11 -04:00
parent c407fb2fc8
commit 399dfc5aa2
22 changed files with 22461 additions and 0 deletions
--- a/.eslintrc.js
+++ b/.eslintrc.js
@@ -0,0 +1,37 @@
+module.exports = {
+  parser: '@typescript-eslint/parser',
+  parserOptions: {
+    project: 'tsconfig.json',
+    sourceType: 'module',
+  },
+  plugins: ['@typescript-eslint/eslint-plugin', 'simple-import-sort'],
+  extends: [
+    'plugin:@typescript-eslint/recommended',
+    'plugin:prettier/recommended',
+  ],
+  root: true,
+  env: {
+    node: true,
+    jest: true,
+  },
+  ignorePatterns: ['.eslintrc.js'],
+  rules: {
+    '@typescript-eslint/interface-name-prefix': 'off',
+    '@typescript-eslint/explicit-function-return-type': 'off',
+    '@typescript-eslint/explicit-module-boundary-types': 'off',
+    '@typescript-eslint/no-explicit-any': 'off',
+    'no-console': 0,
+    'no-unused-vars': 0,
+    '@typescript-eslint/no-unused-vars': [
+      1,
+      {
+        args: 'all',
+        argsIgnorePattern: '^_',
+        varsIgnorePattern: '^_',
+      },
+    ],
+    'no-param-reassign': 0,
+    'simple-import-sort/imports': 'error',
+    'simple-import-sort/exports': 'error',
+  },
+};
--- a/.gitignore
+++ b/.gitignore
@@ -0,0 +1,34 @@
+# compiled output
+/dist
+/node_modules
+
+# Logs
+logs
+*.log
+npm-debug.log*
+yarn-debug.log*
+yarn-error.log*
+lerna-debug.log*
+
+# OS
+.DS_Store
+
+# Tests
+/coverage
+/.nyc_output
+
+# IDEs and editors
+/.idea
+.project
+.classpath
+.c9/
+*.launch
+.settings/
+*.sublime-workspace
+
+# IDE - VSCode
+.vscode/*
+!.vscode/settings.json
+!.vscode/tasks.json
+!.vscode/launch.json
+!.vscode/extensions.json
--- a/.prettierrc
+++ b/.prettierrc
@@ -0,0 +1,4 @@
+{
+  "singleQuote": true,
+  "trailingComma": "all"
+}
--- a/README.md
+++ b/README.md
@@ -0,0 +1,73 @@
+<p align="center">
+  <a href="http://nestjs.com/" target="blank"><img src="https://nestjs.com/img/logo_text.svg" width="320" alt="Nest Logo" /></a>
+</p>
+
+[circleci-image]: https://img.shields.io/circleci/build/github/nestjs/nest/master?token=abc123def456
+[circleci-url]: https://circleci.com/gh/nestjs/nest
+
+  <p align="center">A progressive <a href="http://nodejs.org" target="_blank">Node.js</a> framework for building efficient and scalable server-side applications.</p>
+    <p align="center">
+<a href="https://www.npmjs.com/~nestjscore" target="_blank"><img src="https://img.shields.io/npm/v/@nestjs/core.svg" alt="NPM Version" /></a>
+<a href="https://www.npmjs.com/~nestjscore" target="_blank"><img src="https://img.shields.io/npm/l/@nestjs/core.svg" alt="Package License" /></a>
+<a href="https://www.npmjs.com/~nestjscore" target="_blank"><img src="https://img.shields.io/npm/dm/@nestjs/common.svg" alt="NPM Downloads" /></a>
+<a href="https://circleci.com/gh/nestjs/nest" target="_blank"><img src="https://img.shields.io/circleci/build/github/nestjs/nest/master" alt="CircleCI" /></a>
+<a href="https://coveralls.io/github/nestjs/nest?branch=master" target="_blank"><img src="https://coveralls.io/repos/github/nestjs/nest/badge.svg?branch=master#9" alt="Coverage" /></a>
+<a href="https://discord.gg/G7Qnnhy" target="_blank"><img src="https://img.shields.io/badge/discord-online-brightgreen.svg" alt="Discord"/></a>
+<a href="https://opencollective.com/nest#backer" target="_blank"><img src="https://opencollective.com/nest/backers/badge.svg" alt="Backers on Open Collective" /></a>
+<a href="https://opencollective.com/nest#sponsor" target="_blank"><img src="https://opencollective.com/nest/sponsors/badge.svg" alt="Sponsors on Open Collective" /></a>
+  <a href="https://paypal.me/kamilmysliwiec" target="_blank"><img src="https://img.shields.io/badge/Donate-PayPal-ff3f59.svg"/></a>
+    <a href="https://opencollective.com/nest#sponsor"  target="_blank"><img src="https://img.shields.io/badge/Support%20us-Open%20Collective-41B883.svg" alt="Support us"></a>
+  <a href="https://twitter.com/nestframework" target="_blank"><img src="https://img.shields.io/twitter/follow/nestframework.svg?style=social&label=Follow"></a>
+</p>
+  <!--[![Backers on Open Collective](https://opencollective.com/nest/backers/badge.svg)](https://opencollective.com/nest#backer)
+  [![Sponsors on Open Collective](https://opencollective.com/nest/sponsors/badge.svg)](https://opencollective.com/nest#sponsor)-->
+
+## Description
+
+[Nest](https://github.com/nestjs/nest) framework TypeScript starter repository.
+
+## Installation
+
+```bash
+$ npm install
+```
+
+## Running the app
+
+```bash
+# development
+$ npm run start
+
+# watch mode
+$ npm run start:dev
+
+# production mode
+$ npm run start:prod
+```
+
+## Test
+
+```bash
+# unit tests
+$ npm run test
+
+# e2e tests
+$ npm run test:e2e
+
+# test coverage
+$ npm run test:cov
+```
+
+## Support
+
+Nest is an MIT-licensed open source project. It can grow thanks to the sponsors and support by the amazing backers. If you'd like to join them, please [read more here](https://docs.nestjs.com/support).
+
+## Stay in touch
+
+- Author - [Kamil Myśliwiec](https://kamilmysliwiec.com)
+- Website - [https://nestjs.com](https://nestjs.com/)
+- Twitter - [@nestframework](https://twitter.com/nestframework)
+
+## License
+
+Nest is [MIT licensed](LICENSE).
--- a/nest-cli.json
+++ b/nest-cli.json
@@ -0,0 +1,4 @@
+{
+  "collection": "@nestjs/schematics",
+  "sourceRoot": "src"
+}
--- a/package-lock.json
+++ b/package-lock.json
--- a/package.json
+++ b/package.json
@@ -0,0 +1,83 @@
+{
+  "name": "vidgrab",
+  "version": "0.0.1",
+  "description": "",
+  "author": "",
+  "private": true,
+  "license": "UNLICENSED",
+  "scripts": {
+    "prebuild": "rimraf dist",
+    "build": "nest build",
+    "format": "prettier --write \"src/**/*.ts\" \"test/**/*.ts\"",
+    "start": "nest start",
+    "start:dev": "nest start --watch",
+    "start:debug": "nest start --debug --watch",
+    "start:prod": "node dist/main",
+    "lint": "eslint \"{src,apps,libs,test}/**/*.ts\" --fix",
+    "test": "jest",
+    "test:watch": "jest --watch",
+    "test:cov": "jest --coverage",
+    "test:debug": "node --inspect-brk -r tsconfig-paths/register -r ts-node/register node_modules/.bin/jest --runInBand",
+    "test:e2e": "jest --config ./test/jest-e2e.json"
+  },
+  "dependencies": {
+    "@nestjs/bull": "^0.4.0",
+    "@nestjs/common": "^7.6.15",
+    "@nestjs/core": "^7.6.15",
+    "@nestjs/platform-express": "^7.6.15",
+    "bull": "^3.26.0",
+    "class-transformer": "^0.4.0",
+    "class-validator": "^0.13.1",
+    "express-handlebars": "^5.3.2",
+    "lodash": "^4.17.21",
+    "reflect-metadata": "^0.1.13",
+    "rimraf": "^3.0.2",
+    "rxjs": "^6.6.6",
+    "split2": "^3.2.2",
+    "youtube-dl-exec": "tedkulp/youtube-dl-exec#master"
+  },
+  "devDependencies": {
+    "@nestjs/cli": "^7.6.0",
+    "@nestjs/schematics": "^7.3.0",
+    "@nestjs/testing": "^7.6.15",
+    "@types/bull": "^3.15.2",
+    "@types/express": "^4.17.11",
+    "@types/express-handlebars": "^5.3.1",
+    "@types/jest": "^26.0.22",
+    "@types/lodash": "^4.14.171",
+    "@types/node": "^14.14.36",
+    "@types/split2": "^3.2.1",
+    "@types/supertest": "^2.0.10",
+    "@typescript-eslint/eslint-plugin": "^4.19.0",
+    "@typescript-eslint/parser": "^4.19.0",
+    "eslint": "^7.22.0",
+    "eslint-config-prettier": "^8.1.0",
+    "eslint-plugin-prettier": "^3.3.1",
+    "eslint-plugin-simple-import-sort": "^7.0.0",
+    "jest": "^26.6.3",
+    "prettier": "^2.2.1",
+    "supertest": "^6.1.3",
+    "ts-jest": "^26.5.4",
+    "ts-loader": "^8.0.18",
+    "ts-node": "^9.1.1",
+    "tsconfig-paths": "^3.9.0",
+    "typescript": "^4.2.3"
+  },
+  "jest": {
+    "moduleFileExtensions": [
+      "js",
+      "json",
+      "ts"
+    ],
+    "rootDir": "src",
+    "testRegex": ".*\\.spec\\.ts$",
+    "transform": {
+      "^.+\\.(t|j)s$": "ts-jest"
+    },
+    "collectCoverageFrom": [
+      "**/*.(t|j)s"
+    ],
+    "coverageDirectory": "../coverage",
+    "testEnvironment": "node"
+  }
+}
--- a/src/app.module.ts
+++ b/src/app.module.ts
@@ -0,0 +1,20 @@
+import { BullModule } from '@nestjs/bull';
+import { Module } from '@nestjs/common';
+
+import { WebModule } from './web/web.module';
+import { YtdlModule } from './ytdl/ytdl.module';
+
+@Module({
+  imports: [
+    BullModule.forRoot({
+      redis: {
+        host: 'localhost',
+        port: 6379,
+      },
+      prefix: 'vgqueue',
+    }),
+    YtdlModule,
+    WebModule,
+  ],
+})
+export class AppModule {}
--- a/src/main.ts
+++ b/src/main.ts
@@ -0,0 +1,29 @@
+import { NestFactory } from '@nestjs/core';
+import { NestExpressApplication } from '@nestjs/platform-express';
+import * as exphbs from 'express-handlebars';
+import { capitalize, truncate } from 'lodash';
+import { join } from 'path';
+
+import { AppModule } from './app.module';
+
+async function bootstrap() {
+  const app = await NestFactory.create<NestExpressApplication>(AppModule);
+
+  const hbs = exphbs.create({
+    defaultLayout: 'main',
+    layoutsDir: join(__dirname, '..', 'views', 'layouts'),
+    helpers: {
+      capitalize,
+      truncate: (str: string, options: any) => truncate(str, options.hash),
+    },
+    extname: '.hbs',
+  });
+
+  app.useStaticAssets(join(__dirname, '..', 'public'));
+  app.setBaseViewsDir(join(__dirname, '..', 'views'));
+  app.engine('.hbs', hbs.engine);
+  app.setViewEngine('.hbs');
+
+  await app.listen(3000);
+}
+bootstrap();
--- a/src/types.ts
+++ b/src/types.ts
@@ -0,0 +1,10 @@
+export class UploadDto {
+  url: string;
+}
+
+export class QueueDto {
+  format: number;
+  url: string;
+  title: string;
+  extractor: string;
+}
--- a/src/web/web.controller.ts
+++ b/src/web/web.controller.ts
@@ -0,0 +1,93 @@
+import { InjectQueue } from '@nestjs/bull';
+import {
+  Body,
+  Controller,
+  Get,
+  Logger,
+  Post,
+  Redirect,
+  Render,
+  UsePipes,
+  ValidationPipe,
+} from '@nestjs/common';
+import { Queue } from 'bull';
+import { pick } from 'lodash';
+import { YtdlService } from 'src/ytdl/ytdl.service';
+
+import { QueueDto, UploadDto } from '../types';
+
+@Controller()
+export class WebController {
+  private readonly logger = new Logger(WebController.name);
+
+  constructor(
+    @InjectQueue('vidgrab') private readonly vidgrabQueue: Queue,
+    private readonly ytdlService: YtdlService,
+  ) {}
+
+  @Get()
+  @Render('index')
+  async root() {
+    const jobs = await this.vidgrabQueue.getJobs([
+      'completed',
+      'waiting',
+      'active',
+      'delayed',
+      'failed',
+      'paused',
+    ]);
+
+    const jobList = await Promise.all(
+      jobs.map(async (j) => {
+        const state = await j.getState();
+        const progress = j.progress();
+
+        return {
+          ...j,
+          progress: progress ? `${progress}%` : 'n/a',
+          state: state,
+        };
+      }),
+    );
+
+    return {
+      message: 'Hello world!',
+      jobs: jobList
+        .sort((a, b) => parseInt(b.id.toString()) - parseInt(a.id.toString()))
+        .slice(0, 10),
+    };
+  }
+  s;
+
+  @Post('/getinfo')
+  @Render('getinfo')
+  @UsePipes(new ValidationPipe({ transform: true }))
+  async getInfo(@Body() body: UploadDto) {
+    const videoInfo = await this.ytdlService.getVideoInfo(body.url);
+
+    return {
+      title: videoInfo.title,
+      extractor: videoInfo.extractor,
+      description: videoInfo.description,
+      url: videoInfo.webpage_url,
+      thumbnails: videoInfo.thumbnails,
+      upload_date: videoInfo.upload_date,
+      duration: videoInfo.duration,
+      formats: videoInfo.formats.map((f) =>
+        pick(f, ['format_id', 'format', 'filesize', 'format_note', 'ext']),
+      ),
+    };
+  }
+
+  @Post('/queue')
+  @Redirect('/')
+  @UsePipes(new ValidationPipe({ transform: true }))
+  async queue(@Body() body: QueueDto) {
+    const job = await this.vidgrabQueue.add('download', body);
+
+    // Redirect to main page
+    return {
+      jobId: job.id,
+    };
+  }
+}
--- a/src/web/web.module.ts
+++ b/src/web/web.module.ts
@@ -0,0 +1,16 @@
+import { BullModule } from '@nestjs/bull';
+import { Module } from '@nestjs/common';
+import { YtdlModule } from 'src/ytdl/ytdl.module';
+
+import { WebController } from './web.controller';
+
+@Module({
+  imports: [
+    BullModule.registerQueue({
+      name: 'vidgrab',
+    }),
+    YtdlModule,
+  ],
+  controllers: [WebController],
+})
+export class WebModule {}
--- a/src/ytdl/ytdl.module.ts
+++ b/src/ytdl/ytdl.module.ts
@@ -0,0 +1,10 @@
+import { Module } from '@nestjs/common';
+
+import { YtdlProcessor } from './ytdl.processor';
+import { YtdlService } from './ytdl.service';
+
+@Module({
+  providers: [YtdlProcessor, YtdlService],
+  exports: [YtdlProcessor, YtdlService],
+})
+export class YtdlModule {}
--- a/src/ytdl/ytdl.processor.ts
+++ b/src/ytdl/ytdl.processor.ts
@@ -0,0 +1,97 @@
+import {
+  OnQueueActive,
+  OnQueueCompleted,
+  OnQueueError,
+  OnQueueFailed,
+  OnQueueProgress,
+  Process,
+  Processor,
+} from '@nestjs/bull';
+import { Logger } from '@nestjs/common';
+import { Job } from 'bull';
+import { throttle } from 'lodash';
+import * as split from 'split2';
+import { QueueDto } from 'src/types';
+import { raw } from 'youtube-dl-exec';
+
+@Processor('vidgrab')
+export class YtdlProcessor {
+  private readonly logger = new Logger(YtdlProcessor.name);
+  private setProgress = throttle((job: Job, progress: number) => {
+    job.progress(progress);
+  }, 250);
+
+  @Process('download')
+  async handleDownload(job: Job<QueueDto>) {
+    this.logger.debug('Start downloading...');
+    this.logger.debug(job.data);
+
+    try {
+      const execaProcess = raw(job.data.url, {
+        format: job.data.format,
+        output: '/tmp/%(title)s-%(format_id)s.%(ext)s',
+        newline: true,
+      });
+
+      execaProcess.stdout.pipe(split()).on('data', (line) => {
+        this.logger.verbose(`youtube-dl stdout: ${line}`);
+
+        const percentDone = line.match(/\[download\]\s+([0-9.]+)% of/);
+        if (percentDone) {
+          this.setProgress(job, percentDone[1]);
+        }
+      });
+
+      execaProcess.stderr.pipe(split()).on('data', (line) => {
+        this.logger.error(`youtube-dl stderr: ${line}`);
+      });
+
+      await execaProcess.then();
+    } catch (err) {
+      this.logger.error(err);
+    }
+
+    this.logger.debug('Downloading completed');
+  }
+
+  @OnQueueActive()
+  onActive(job: Job) {
+    this.logger.debug(
+      `Processing job ${job.id} of type ${job.name} with data ${JSON.stringify(
+        job.data,
+      )}...`,
+    );
+  }
+
+  @OnQueueCompleted()
+  onCompleted(job: Job, result: any) {
+    this.logger.debug(
+      `Completed job ${job.id} of type ${job.name} with data ${JSON.stringify(
+        job.data,
+      )} and result ${result}...`,
+    );
+  }
+
+  @OnQueueFailed()
+  onFailed(job: Job, err: Error) {
+    this.logger.error(
+      `Failed job ${job.id} of type ${job.name} with data ${JSON.stringify(
+        job.data,
+      )} and error ${err}...`,
+    );
+  }
+
+  @OnQueueError()
+  onError(err: Error) {
+    this.logger.error(`Error is ${err}...`);
+  }
+
+  @OnQueueProgress()
+  onProgress(job: Job, progress: number) {
+    this.logger.verbose(
+      `Progress on ${job.id} of type ${job.name} with data ${JSON.stringify(
+        job.data,
+      )} and progress ${progress}...`,
+    );
+  }
+}
--- a/src/ytdl/ytdl.service.ts
+++ b/src/ytdl/ytdl.service.ts
@@ -0,0 +1,18 @@
+import { Injectable, Logger } from '@nestjs/common';
+import { default as ytdl } from 'youtube-dl-exec';
+
+@Injectable()
+export class YtdlService {
+  private readonly logger = new Logger(YtdlService.name);
+
+  async getVideoInfo(url: string) {
+    return ytdl(url, {
+      dumpSingleJson: true,
+      noWarnings: true,
+      noCallHome: true,
+      noCheckCertificate: true,
+      preferFreeFormats: true,
+      youtubeSkipDashManifest: true,
+    });
+  }
+}
--- a/test/app.e2e-spec.ts
+++ b/test/app.e2e-spec.ts
@@ -0,0 +1,24 @@
+import { Test, TestingModule } from '@nestjs/testing';
+import { INestApplication } from '@nestjs/common';
+import * as request from 'supertest';
+import { AppModule } from './../src/app.module';
+
+describe('AppController (e2e)', () => {
+  let app: INestApplication;
+
+  beforeEach(async () => {
+    const moduleFixture: TestingModule = await Test.createTestingModule({
+      imports: [AppModule],
+    }).compile();
+
+    app = moduleFixture.createNestApplication();
+    await app.init();
+  });
+
+  it('/ (GET)', () => {
+    return request(app.getHttpServer())
+      .get('/')
+      .expect(200)
+      .expect('Hello World!');
+  });
+});
--- a/test/jest-e2e.json
+++ b/test/jest-e2e.json
@@ -0,0 +1,9 @@
+{
+  "moduleFileExtensions": ["js", "json", "ts"],
+  "rootDir": ".",
+  "testEnvironment": "node",
+  "testRegex": ".e2e-spec.ts$",
+  "transform": {
+    "^.+\\.(t|j)s$": "ts-jest"
+  }
+}
--- a/tsconfig.build.json
+++ b/tsconfig.build.json
@@ -0,0 +1,4 @@
+{
+  "extends": "./tsconfig.json",
+  "exclude": ["node_modules", "test", "dist", "**/*spec.ts"]
+}
--- a/tsconfig.json
+++ b/tsconfig.json
@@ -0,0 +1,15 @@
+{
+  "compilerOptions": {
+    "module": "commonjs",
+    "declaration": true,
+    "removeComments": true,
+    "emitDecoratorMetadata": true,
+    "experimentalDecorators": true,
+    "allowSyntheticDefaultImports": true,
+    "target": "es2017",
+    "sourceMap": true,
+    "outDir": "./dist",
+    "baseUrl": "./",
+    "incremental": true
+  }
+}
--- a/views/getinfo.hbs
+++ b/views/getinfo.hbs
@@ -0,0 +1,24 @@
+<section class="section">
+  <div class="container">
+    <form method="POST" action="/queue">
+      <div class="field is-grouped">
+        <div class="control is-expanded">
+          <div class="select is-fullwidth">
+            <select name="format">
+              {{#each formats}}
+              <option value={{format_id}}>{{format}} ({{ext}})</option>
+              {{/each}}
+            </select>
+          </div>
+        </div>
+
+        <div class="control">
+          <input class="button is-primary" type="submit" value="Queue Download" />
+        </div>
+      </div>
+      <input type="hidden" name="url" value="{{url}}" />
+      <input type="hidden" name="title" value="{{title}}" />
+      <input type="hidden" name="extractor" value="{{extractor}}" />
+    </form>
+  </div>
+</section>
--- a/views/index.hbs
+++ b/views/index.hbs
@@ -0,0 +1,45 @@
+<section class="section">
+  <div class="container">
+    <form method="POST" action="/getinfo">
+      <div class="field is-grouped">
+        <div class="control is-expanded">
+          <input class="input is-medium" name="url" placeholder="URL to Download" />
+        </div>
+        <div class="control">
+          <input class="button is-primary is-medium" type="submit" value="Submit" />
+        </div>
+      </div>
+    </form>
+  </div>
+</section>
+
+<section class="section">
+  <div class="container">
+    <h2 class="subtitle is-4 has-text-centered">Recent Downloads</h2>
+
+    <table class="table is-striped is-hoverable is-fullwidth is-narrow">
+      <thead>
+        <tr>
+          <th>ID</th>
+          <th>Status</th>
+          <th><abbr title="Service">Svc</abbr></th>
+          <th>Title</th>
+          <th><abbr title="Percent Downloaded">%</abbr></th>
+        </tr>
+      </thead>
+      {{#each jobs}}
+      <tr>
+        <td>{{id}}</td>
+        <td>{{capitalize state}}</td>
+        <td>{{capitalize data.extractor}}</td>
+        <td>
+          <a href="{{data.url}}" title="{{data.title}}" target="_blank" rel="noopener noreferrer">
+            {{data.title}}
+          </a>
+        </td>
+        <td>{{progress}}</td>
+      </tr>
+      {{/each}}
+    </table>
+  </div>
+</section>
--- a/views/layouts/main.hbs
+++ b/views/layouts/main.hbs
@@ -0,0 +1,50 @@
+<!DOCTYPE html>
+<html>
+
+<head>
+  <meta charset="utf-8">
+  <meta name="viewport" content="width=device-width, initial-scale=1">
+  <title>Vidgrab</title>
+  <link rel="stylesheet" href="https://cdn.jsdelivr.net/npm/bulma@0.9.3/css/bulma.min.css">
+  <style type="text/css" media="screen">
+    body {
+      display: flex;
+      min-height: 100vh;
+      flex-direction: column;
+    }
+
+    #wrapper {
+      flex: 1;
+    }
+
+    #footer {
+      padding-top: 2em;
+      padding-bottom: 2em;
+    }
+  </style>
+</head>
+
+<body>
+  <div id="wrapper">
+    <section class="section has-background-light">
+      <div class="container">
+        <nav class="level">
+          <h1 class="title is-1 level-item has-text-centered">Vidgrab</h1>
+        </nav>
+        <div>
+    </section>
+
+    {{{body}}}
+
+  </div>
+
+  <footer id="footer" class="footer">
+    <div class="content has-text-centered">
+      <p>
+        Code and "Design" by <a href="#">Ted Kulp</a>. Based on youtube-dl.
+      </p>
+    </div>
+  </footer>
+</body>
+
+</html>