Got the basics working with angular

.gitignore

@@ -37,3 +37,7 @@ testem.log
 # System Files
 .DS_Store
 Thumbs.db
+
+# Custom stuff
+/package
+/*.tgz

apps/api/src/app/app.module.ts

@@ -1,15 +1,41 @@
 import { Module } from '@nestjs/common';
 import { ServeStaticModule } from '@nestjs/serve-static';
 import { join } from 'path';
+import { ConfigModule, ConfigService } from '@nestjs/config';
+import { EventEmitterModule } from '@nestjs/event-emitter';
+import { BullModule } from '@nestjs/bull';
 
 import { AppController } from './app.controller';
 import { AppService } from './app.service';
+import { WebModule } from '../web/web.module';
+import { YtdlModule } from '../ytdl/ytdl.module';
 
 @Module({
   imports: [
+    ConfigModule.forRoot({
+      ignoreEnvFile: true,
+      isGlobal: true,
+      // load: [configuration],
+    }),
+    EventEmitterModule.forRoot({
+      wildcard: true,
+    }),
+    BullModule.forRootAsync({
+      imports: [ConfigModule],
+      useFactory: (configService: ConfigService) => ({
+        redis: {
+          host: configService.get('redisHost'),
+          port: configService.get('redisPort'),
+        },
+        prefix: 'vgqueue',
+      }),
+      inject: [ConfigService],
+    }),
     ServeStaticModule.forRoot({
       rootPath: join(__dirname, '..', 'client'),
     }),
+    YtdlModule,
+    WebModule,
   ],
   controllers: [AppController],
   providers: [AppService],

apps/api/src/main.ts

@@ -5,11 +5,12 @@
 
 import { Logger } from '@nestjs/common';
 import { NestFactory } from '@nestjs/core';
+import { NestExpressApplication } from '@nestjs/platform-express';
 
 import { AppModule } from './app/app.module';
 
 async function bootstrap() {
-  const app = await NestFactory.create(AppModule);
+  const app = await NestFactory.create<NestExpressApplication>(AppModule);
   const globalPrefix = 'api';
   app.setGlobalPrefix(globalPrefix);
   const port = process.env.PORT || 3333;

apps/api/src/web/web.controller.ts

@@ -0,0 +1,123 @@
+import { InjectQueue } from '@nestjs/bull';
+import {
+  Body,
+  Controller,
+  Get,
+  HttpCode,
+  Logger,
+  Post,
+  Render,
+  Req,
+  UsePipes,
+  ValidationPipe,
+} from '@nestjs/common';
+import { EventEmitter2 } from '@nestjs/event-emitter';
+import { Queue } from 'bull';
+import { pick, trimEnd } from 'lodash';
+
+import { QueueDto, UploadDto } from '@vidgrab2/api-interfaces';
+import { YtdlService } from '../ytdl/ytdl.service';
+
+@Controller()
+export class WebController {
+  private readonly logger = new Logger(WebController.name);
+
+  constructor(
+    @InjectQueue('vidgrab') private readonly vidgrabQueue: Queue,
+    private readonly ytdlService: YtdlService,
+    private readonly eventEmitter: EventEmitter2,
+  ) {}
+
+  @Get()
+  @Render('Index')
+  async root(@Req() request: any) {
+    const fullUrl =
+      request.protocol + '://' + request.get('host') + request.originalUrl;
+    const bookmarklet = `javascript:(function(){var xhr=new XMLHttpRequest();xhr.open('POST',encodeURI('${trimEnd(
+      fullUrl,
+      '/',
+    )}/queue'));xhr.setRequestHeader('Content-Type','application/x-www-form-urlencoded');xhr.send('url='+document.location.href.replace(/ /g,'+'));}());`;
+
+    const jobs = await this.vidgrabQueue.getJobs([
+      'completed',
+      'waiting',
+      'active',
+      'delayed',
+      'failed',
+      'paused',
+    ]);
+
+    const jobList = await Promise.all(
+      jobs.map(async (j) => {
+        const state = await j.getState();
+        const progress = j.progress();
+
+        return {
+          ...pick(j, ['id', 'name', 'data']),
+          progress: progress ? `${progress}%` : 'n/a',
+          state: state,
+        };
+      }),
+    );
+
+    return {
+      bookmarklet,
+      jobs: JSON.parse(
+        JSON.stringify(
+          jobList
+            .sort(
+              (a: {id: string}, b: { id: string }) => parseInt(b.id.toString()) - parseInt(a.id.toString()),
+            )
+            .slice(0, 10),
+        ),
+      ),
+    };
+  }
+
+  @Get('/extractors')
+  async listExtractors() {
+    const extractors = await this.ytdlService.listExtractors();
+
+    return {
+      extractors,
+    };
+  }
+
+  @Post('/getinfo')
+  @HttpCode(200)
+  @UsePipes(new ValidationPipe({ transform: true }))
+  async getInfo(@Body() body: UploadDto) {
+    const videoInfo = await this.ytdlService.getVideoInfo(body.url);
+
+    return {
+      title: videoInfo.title,
+      extractor: videoInfo.extractor,
+      description: videoInfo.description,
+      videoUrl: videoInfo.webpage_url,
+      thumbnails: videoInfo.thumbnails,
+      upload_date: videoInfo.upload_date,
+      duration: videoInfo.duration,
+      formats: videoInfo.formats.map((f) =>
+        pick(f, ['format_id', 'format', 'filesize', 'format_note', 'ext']),
+      ),
+    };
+  }
+
+  @Post('/queue')
+  @UsePipes(new ValidationPipe({ transform: true }))
+  async queue(@Body() body: QueueDto) {
+    if (!body.extractor || !body.title) {
+      const videoInfo = await this.ytdlService.getVideoInfo(body.url);
+      body.extractor = videoInfo.extractor;
+      body.title = videoInfo.title;
+    }
+
+    const job = await this.vidgrabQueue.add('download', body);
+
+    this.eventEmitter.emit('job.added', { job });
+
+    return {
+      jobId: job.id,
+    };
+  }
+}

apps/api/src/web/web.module.ts

@@ -0,0 +1,16 @@
+import { BullModule } from '@nestjs/bull';
+import { Module } from '@nestjs/common';
+import { YtdlModule } from '../ytdl/ytdl.module';
+
+import { WebController } from './web.controller';
+
+@Module({
+  imports: [
+    BullModule.registerQueue({
+      name: 'vidgrab',
+    }),
+    YtdlModule,
+  ],
+  controllers: [WebController],
+})
+export class WebModule {}

apps/api/src/ytdl/ytdl.module.ts

@@ -0,0 +1,9 @@
+import { Module } from '@nestjs/common';
+import { YtdlService } from './ytdl.service';
+import { YtdlProcessor } from './ytdl.processor';
+
+@Module({
+  providers: [YtdlProcessor, YtdlService],
+  exports: [YtdlProcessor, YtdlService],
+})
+export class YtdlModule {}

apps/api/src/ytdl/ytdl.processor.ts

@@ -0,0 +1,116 @@
+import {
+  OnQueueActive,
+  OnQueueCompleted,
+  OnQueueError,
+  OnQueueFailed,
+  OnQueueProgress,
+  Process,
+  Processor,
+} from '@nestjs/bull';
+import { Logger } from '@nestjs/common';
+import { ConfigService } from '@nestjs/config';
+import { EventEmitter2 } from '@nestjs/event-emitter';
+import { Job } from 'bull';
+import { throttle } from 'lodash';
+import split from 'split2';
+import { QueueDto } from '@vidgrab2/api-interfaces';
+import { raw } from 'youtube-dl-exec';
+
+@Processor('vidgrab')
+export class YtdlProcessor {
+  private readonly logger = new Logger(YtdlProcessor.name);
+  private setProgress = throttle((job: Job, progress: number) => {
+    job.progress(progress);
+  }, 250);
+
+  constructor(
+    private readonly configService: ConfigService,
+    private readonly eventEmitter: EventEmitter2,
+  ) {}
+
+  @Process('download')
+  async handleDownload(job: Job<QueueDto>) {
+    this.logger.debug('Start downloading...');
+    this.logger.debug(job.data);
+
+    try {
+      const execaProcess = raw(job.data.url, {
+        format: job.data.format,
+        output: `${this.configService.get(
+          'fileDir',
+        )}/%(title)s-%(format_id)s.%(ext)s`,
+        newline: true,
+      });
+
+      execaProcess.stdout?.pipe(split()).on('data', (line: string) => {
+        this.logger.verbose(`youtube-dl stdout: ${line}`);
+
+        const percentDone = line.match(/\[download\]\s+([0-9.]+)% of/);
+        if (percentDone) {
+          this.setProgress(job, parseFloat(percentDone[1]));
+        }
+      });
+
+      execaProcess.stderr?.pipe(split()).on('data', (line: any) => {
+        this.logger.error(`youtube-dl stderr: ${line}`);
+      });
+
+      await execaProcess.then();
+    } catch (err) {
+      this.logger.error(err);
+    }
+
+    this.logger.debug('Downloading completed');
+  }
+
+  @OnQueueActive()
+  onActive(job: Job) {
+    this.logger.debug(
+      `Processing job ${job.id} of type ${job.name} with data ${JSON.stringify(
+        job.data,
+      )}...`,
+    );
+
+    this.eventEmitter.emit('job.updated', { job });
+  }
+
+  @OnQueueCompleted()
+  onCompleted(job: Job, result: any) {
+    this.logger.debug(
+      `Completed job ${job.id} of type ${job.name} with data ${JSON.stringify(
+        job.data,
+      )} and result ${result}...`,
+    );
+
+    this.eventEmitter.emit('job.updated', { job });
+  }
+
+  @OnQueueFailed()
+  onFailed(job: Job, err: Error) {
+    this.logger.error(
+      `Failed job ${job.id} of type ${job.name} with data ${JSON.stringify(
+        job.data,
+      )} and error ${err}...`,
+    );
+
+    this.eventEmitter.emit('job.failed', { job, error: err });
+  }
+
+  @OnQueueError()
+  onError(err: Error) {
+    this.logger.error(`Error is ${err}...`);
+
+    this.eventEmitter.emit('job.error', { error: err });
+  }
+
+  @OnQueueProgress()
+  onProgress(job: Job, progress: number) {
+    this.logger.verbose(
+      `Progress on ${job.id} of type ${job.name} with data ${JSON.stringify(
+        job.data,
+      )} and progress ${progress}...`,
+    );
+
+    this.eventEmitter.emit('job.updated', { job });
+  }
+}

apps/api/src/ytdl/ytdl.service.ts

@@ -0,0 +1,26 @@
+import { Injectable, Logger } from '@nestjs/common';
+import { default as ytdl } from 'youtube-dl-exec';
+
+@Injectable()
+export class YtdlService {
+  private readonly logger = new Logger(YtdlService.name);
+
+  async getVideoInfo(url: string) {
+    return ytdl(url, {
+      dumpSingleJson: true,
+      noWarnings: true,
+      noCallHome: true,
+      noCheckCertificate: true,
+      preferFreeFormats: true,
+      youtubeSkipDashManifest: true,
+    });
+  }
+
+  async listExtractors() {
+    const extractors = (await ytdl('', {
+      listExtractors: true,
+    })) as unknown;
+
+    return (extractors as string).split('\n');
+  }
+}

apps/client/src/app/app.component.html

@@ -1,4 +1,4 @@
-<div style="text-align: center">
+<!-- <div style="text-align: center">
   <h1>Welcome to client!</h1>
   <img
     width="450"
@@ -6,4 +6,5 @@
     alt="Nx - Smart, Extensible Build Framework"
   />
 </div>
-<div>Message: {{ hello$ | async | json }}</div>
+<div>Message: {{ hello$ | async | json }}</div> -->
+<router-outlet></router-outlet>

apps/client/src/app/app.module.ts

@@ -1,12 +1,19 @@
 import { NgModule } from '@angular/core';
 import { BrowserModule } from '@angular/platform-browser';
+import { RouterModule, Routes } from '@angular/router'
 
 import { AppComponent } from './app.component';
 import { HttpClientModule } from '@angular/common/http';
+import { ExtractorsComponent } from './extractors/extractors.component';
+
+const routes: Routes = [
+  { path: '', component: AppComponent },
+  { path: 'extractors', component: ExtractorsComponent },
+];
 
 @NgModule({
-  declarations: [AppComponent],
-  imports: [BrowserModule, HttpClientModule],
+  declarations: [AppComponent, ExtractorsComponent],
+  imports: [BrowserModule, HttpClientModule, RouterModule.forRoot(routes)],
   providers: [],
   bootstrap: [AppComponent],
 })

apps/client/src/app/extractors/extractors.component.html

@@ -0,0 +1,13 @@
+<section class="section">
+  <div class="container">
+    <div class="content">
+      <p>
+        Here is the list of currently supported services. This is generated
+        dynamically because youtube-dl adds new services all the time.
+      </p>
+      <ul>
+        <li *ngFor="let e of extractors$ | async">{{ e }}</li>
+      </ul>
+    </div>
+  </div>
+</section>

apps/client/src/app/extractors/extractors.component.ts

@@ -0,0 +1,20 @@
+/* eslint-disable @angular-eslint/no-empty-lifecycle-method */
+import { Component, OnInit } from '@angular/core';
+import { HttpClient } from '@angular/common/http';
+import { map } from 'rxjs/operators';
+
+@Component({
+  selector: 'vidgrab2-extractors',
+  templateUrl: './extractors.component.html',
+  styleUrls: ['./extractors.component.scss']
+})
+export class ExtractorsComponent implements OnInit {
+
+  extractors$ = this.http.get<{extractors: string[]}>('/api/extractors').pipe(map(e => e.extractors));
+  constructor(private http: HttpClient) {}
+
+  ngOnInit(): void {
+    // Does a thing
+  }
+
+}

apps/client/src/index.html

@@ -6,8 +6,30 @@
     <base href="/" />
     <meta name="viewport" content="width=device-width, initial-scale=1" />
     <link rel="icon" type="image/x-icon" href="favicon.ico" />
+    <link
+      rel="stylesheet"
+      href="https://cdn.jsdelivr.net/npm/bulma@0.9.3/css/bulma.min.css"
+    />
   </head>
   <body>
+    <div id="wrapper">
+      <section class="section has-background-light">
+        <div class="container">
+          <nav class="level">
+            <h1 class="title is-1 level-item has-text-centered">Vidgrab</h1>
+          </nav>
+        </div>
+      </section>
+
       <vidgrab2-root></vidgrab2-root>
+    </div>
+
+    <footer id="footer" class="footer">
+      <div class="content has-text-centered">
+        <p>
+          Code and "Design" by <a href="#">Ted Kulp</a>. Based on youtube-dl.
+        </p>
+      </div>
+    </footer>
   </body>
 </html>

apps/client/src/styles.scss

@@ -1 +1,20 @@
 /* You can add global styles to this file, and also import other style files */
+body {
+  display: flex;
+  min-height: 100vh;
+  flex-direction: column;
+}
+
+#wrapper {
+  flex: 1;
+}
+
+#footer {
+  padding-top: 2em;
+  padding-bottom: 2em;
+}
+
+.bookmarklet {
+  border: 1px dashed black;
+  padding: 10px 20px 10px 20px;
+}

libs/api-interfaces/src/lib/api-interfaces.ts

@@ -1,3 +1,19 @@
 export interface Message {
   message: string;
 }
+
+export interface UploadDto {
+  url: string;
+}
+
+export interface QueueDto {
+  format: string;
+  url: string;
+  title?: string;
+  extractor?: string;
+}
+
+export type JobEvent = {
+  job?: unknown;
+  err?: Error;
+};

package.json

@@ -40,14 +40,25 @@
     "@angular/platform-browser": "^12.1.0",
     "@angular/platform-browser-dynamic": "^12.1.0",
     "@angular/router": "^12.1.0",
+    "@nestjs/bull": "^0.4.0",
     "@nestjs/common": "^7.0.0",
+    "@nestjs/config": "^1.0.1",
     "@nestjs/core": "^7.0.0",
+    "@nestjs/event-emitter": "^1.0.0",
     "@nestjs/platform-express": "^7.0.0",
+    "@nestjs/platform-socket.io": "^7.6.18",
     "@nestjs/serve-static": "^2.2.2",
+    "@nestjs/websockets": "^7.6.18",
     "@nrwl/angular": "12.6.3",
+    "bull": "^3.27.0",
+    "class-transformer": "^0.4.0",
+    "class-validator": "^0.13.1",
+    "lodash": "^4.17.21",
     "reflect-metadata": "^0.1.13",
     "rxjs": "~6.6.0",
+    "split2": "^3.2.2",
     "tslib": "^2.0.0",
+    "youtube-dl-exec": "github:tedkulp/youtube-dl-exec#master",
     "zone.js": "~0.11.4"
   },
   "devDependencies": {
@@ -65,7 +76,7 @@
     "@nrwl/eslint-plugin-nx": "12.6.3",
     "@nrwl/jest": "12.6.3",
     "@nrwl/linter": "12.6.3",
-    "@nrwl/nest": "12.6.3",
+    "@nrwl/nest": "^12.6.3",
     "@nrwl/node": "12.6.3",
     "@nrwl/tao": "12.6.3",
     "@nrwl/workspace": "12.6.3",