Got the basics working with angular

apps/api/src/app/app.module.ts

@@ -1,15 +1,41 @@
 import { Module } from '@nestjs/common';
 import { ServeStaticModule } from '@nestjs/serve-static';
 import { join } from 'path';
+import { ConfigModule, ConfigService } from '@nestjs/config';
+import { EventEmitterModule } from '@nestjs/event-emitter';
+import { BullModule } from '@nestjs/bull';
 
 import { AppController } from './app.controller';
 import { AppService } from './app.service';
+import { WebModule } from '../web/web.module';
+import { YtdlModule } from '../ytdl/ytdl.module';
 
 @Module({
   imports: [
+    ConfigModule.forRoot({
+      ignoreEnvFile: true,
+      isGlobal: true,
+      // load: [configuration],
+    }),
+    EventEmitterModule.forRoot({
+      wildcard: true,
+    }),
+    BullModule.forRootAsync({
+      imports: [ConfigModule],
+      useFactory: (configService: ConfigService) => ({
+        redis: {
+          host: configService.get('redisHost'),
+          port: configService.get('redisPort'),
+        },
+        prefix: 'vgqueue',
+      }),
+      inject: [ConfigService],
+    }),
     ServeStaticModule.forRoot({
       rootPath: join(__dirname, '..', 'client'),
     }),
+    YtdlModule,
+    WebModule,
   ],
   controllers: [AppController],
   providers: [AppService],

apps/api/src/main.ts

@@ -5,11 +5,12 @@
 
 import { Logger } from '@nestjs/common';
 import { NestFactory } from '@nestjs/core';
+import { NestExpressApplication } from '@nestjs/platform-express';
 
 import { AppModule } from './app/app.module';
 
 async function bootstrap() {
-  const app = await NestFactory.create(AppModule);
+  const app = await NestFactory.create<NestExpressApplication>(AppModule);
   const globalPrefix = 'api';
   app.setGlobalPrefix(globalPrefix);
   const port = process.env.PORT || 3333;

apps/api/src/web/web.controller.ts

@@ -0,0 +1,123 @@
+import { InjectQueue } from '@nestjs/bull';
+import {
+  Body,
+  Controller,
+  Get,
+  HttpCode,
+  Logger,
+  Post,
+  Render,
+  Req,
+  UsePipes,
+  ValidationPipe,
+} from '@nestjs/common';
+import { EventEmitter2 } from '@nestjs/event-emitter';
+import { Queue } from 'bull';
+import { pick, trimEnd } from 'lodash';
+
+import { QueueDto, UploadDto } from '@vidgrab2/api-interfaces';
+import { YtdlService } from '../ytdl/ytdl.service';
+
+@Controller()
+export class WebController {
+  private readonly logger = new Logger(WebController.name);
+
+  constructor(
+    @InjectQueue('vidgrab') private readonly vidgrabQueue: Queue,
+    private readonly ytdlService: YtdlService,
+    private readonly eventEmitter: EventEmitter2,
+  ) {}
+
+  @Get()
+  @Render('Index')
+  async root(@Req() request: any) {
+    const fullUrl =
+      request.protocol + '://' + request.get('host') + request.originalUrl;
+    const bookmarklet = `javascript:(function(){var xhr=new XMLHttpRequest();xhr.open('POST',encodeURI('${trimEnd(
+      fullUrl,
+      '/',
+    )}/queue'));xhr.setRequestHeader('Content-Type','application/x-www-form-urlencoded');xhr.send('url='+document.location.href.replace(/ /g,'+'));}());`;
+
+    const jobs = await this.vidgrabQueue.getJobs([
+      'completed',
+      'waiting',
+      'active',
+      'delayed',
+      'failed',
+      'paused',
+    ]);
+
+    const jobList = await Promise.all(
+      jobs.map(async (j) => {
+        const state = await j.getState();
+        const progress = j.progress();
+
+        return {
+          ...pick(j, ['id', 'name', 'data']),
+          progress: progress ? `${progress}%` : 'n/a',
+          state: state,
+        };
+      }),
+    );
+
+    return {
+      bookmarklet,
+      jobs: JSON.parse(
+        JSON.stringify(
+          jobList
+            .sort(
+              (a: {id: string}, b: { id: string }) => parseInt(b.id.toString()) - parseInt(a.id.toString()),
+            )
+            .slice(0, 10),
+        ),
+      ),
+    };
+  }
+
+  @Get('/extractors')
+  async listExtractors() {
+    const extractors = await this.ytdlService.listExtractors();
+
+    return {
+      extractors,
+    };
+  }
+
+  @Post('/getinfo')
+  @HttpCode(200)
+  @UsePipes(new ValidationPipe({ transform: true }))
+  async getInfo(@Body() body: UploadDto) {
+    const videoInfo = await this.ytdlService.getVideoInfo(body.url);
+
+    return {
+      title: videoInfo.title,
+      extractor: videoInfo.extractor,
+      description: videoInfo.description,
+      videoUrl: videoInfo.webpage_url,
+      thumbnails: videoInfo.thumbnails,
+      upload_date: videoInfo.upload_date,
+      duration: videoInfo.duration,
+      formats: videoInfo.formats.map((f) =>
+        pick(f, ['format_id', 'format', 'filesize', 'format_note', 'ext']),
+      ),
+    };
+  }
+
+  @Post('/queue')
+  @UsePipes(new ValidationPipe({ transform: true }))
+  async queue(@Body() body: QueueDto) {
+    if (!body.extractor || !body.title) {
+      const videoInfo = await this.ytdlService.getVideoInfo(body.url);
+      body.extractor = videoInfo.extractor;
+      body.title = videoInfo.title;
+    }
+
+    const job = await this.vidgrabQueue.add('download', body);
+
+    this.eventEmitter.emit('job.added', { job });
+
+    return {
+      jobId: job.id,
+    };
+  }
+}

apps/api/src/web/web.module.ts

@@ -0,0 +1,16 @@
+import { BullModule } from '@nestjs/bull';
+import { Module } from '@nestjs/common';
+import { YtdlModule } from '../ytdl/ytdl.module';
+
+import { WebController } from './web.controller';
+
+@Module({
+  imports: [
+    BullModule.registerQueue({
+      name: 'vidgrab',
+    }),
+    YtdlModule,
+  ],
+  controllers: [WebController],
+})
+export class WebModule {}

apps/api/src/ytdl/ytdl.module.ts

@@ -0,0 +1,9 @@
+import { Module } from '@nestjs/common';
+import { YtdlService } from './ytdl.service';
+import { YtdlProcessor } from './ytdl.processor';
+
+@Module({
+  providers: [YtdlProcessor, YtdlService],
+  exports: [YtdlProcessor, YtdlService],
+})
+export class YtdlModule {}

apps/api/src/ytdl/ytdl.processor.ts

@@ -0,0 +1,116 @@
+import {
+  OnQueueActive,
+  OnQueueCompleted,
+  OnQueueError,
+  OnQueueFailed,
+  OnQueueProgress,
+  Process,
+  Processor,
+} from '@nestjs/bull';
+import { Logger } from '@nestjs/common';
+import { ConfigService } from '@nestjs/config';
+import { EventEmitter2 } from '@nestjs/event-emitter';
+import { Job } from 'bull';
+import { throttle } from 'lodash';
+import split from 'split2';
+import { QueueDto } from '@vidgrab2/api-interfaces';
+import { raw } from 'youtube-dl-exec';
+
+@Processor('vidgrab')
+export class YtdlProcessor {
+  private readonly logger = new Logger(YtdlProcessor.name);
+  private setProgress = throttle((job: Job, progress: number) => {
+    job.progress(progress);
+  }, 250);
+
+  constructor(
+    private readonly configService: ConfigService,
+    private readonly eventEmitter: EventEmitter2,
+  ) {}
+
+  @Process('download')
+  async handleDownload(job: Job<QueueDto>) {
+    this.logger.debug('Start downloading...');
+    this.logger.debug(job.data);
+
+    try {
+      const execaProcess = raw(job.data.url, {
+        format: job.data.format,
+        output: `${this.configService.get(
+          'fileDir',
+        )}/%(title)s-%(format_id)s.%(ext)s`,
+        newline: true,
+      });
+
+      execaProcess.stdout?.pipe(split()).on('data', (line: string) => {
+        this.logger.verbose(`youtube-dl stdout: ${line}`);
+
+        const percentDone = line.match(/\[download\]\s+([0-9.]+)% of/);
+        if (percentDone) {
+          this.setProgress(job, parseFloat(percentDone[1]));
+        }
+      });
+
+      execaProcess.stderr?.pipe(split()).on('data', (line: any) => {
+        this.logger.error(`youtube-dl stderr: ${line}`);
+      });
+
+      await execaProcess.then();
+    } catch (err) {
+      this.logger.error(err);
+    }
+
+    this.logger.debug('Downloading completed');
+  }
+
+  @OnQueueActive()
+  onActive(job: Job) {
+    this.logger.debug(
+      `Processing job ${job.id} of type ${job.name} with data ${JSON.stringify(
+        job.data,
+      )}...`,
+    );
+
+    this.eventEmitter.emit('job.updated', { job });
+  }
+
+  @OnQueueCompleted()
+  onCompleted(job: Job, result: any) {
+    this.logger.debug(
+      `Completed job ${job.id} of type ${job.name} with data ${JSON.stringify(
+        job.data,
+      )} and result ${result}...`,
+    );
+
+    this.eventEmitter.emit('job.updated', { job });
+  }
+
+  @OnQueueFailed()
+  onFailed(job: Job, err: Error) {
+    this.logger.error(
+      `Failed job ${job.id} of type ${job.name} with data ${JSON.stringify(
+        job.data,
+      )} and error ${err}...`,
+    );
+
+    this.eventEmitter.emit('job.failed', { job, error: err });
+  }
+
+  @OnQueueError()
+  onError(err: Error) {
+    this.logger.error(`Error is ${err}...`);
+
+    this.eventEmitter.emit('job.error', { error: err });
+  }
+
+  @OnQueueProgress()
+  onProgress(job: Job, progress: number) {
+    this.logger.verbose(
+      `Progress on ${job.id} of type ${job.name} with data ${JSON.stringify(
+        job.data,
+      )} and progress ${progress}...`,
+    );
+
+    this.eventEmitter.emit('job.updated', { job });
+  }
+}

apps/api/src/ytdl/ytdl.service.ts

@@ -0,0 +1,26 @@
+import { Injectable, Logger } from '@nestjs/common';
+import { default as ytdl } from 'youtube-dl-exec';
+
+@Injectable()
+export class YtdlService {
+  private readonly logger = new Logger(YtdlService.name);
+
+  async getVideoInfo(url: string) {
+    return ytdl(url, {
+      dumpSingleJson: true,
+      noWarnings: true,
+      noCallHome: true,
+      noCheckCertificate: true,
+      preferFreeFormats: true,
+      youtubeSkipDashManifest: true,
+    });
+  }
+
+  async listExtractors() {
+    const extractors = (await ytdl('', {
+      listExtractors: true,
+    })) as unknown;
+
+    return (extractors as string).split('\n');
+  }
+}