Move to ESM, use ytsr, implement caching

Closes #315
This commit is contained in:
Max Isom 2021-09-19 22:04:34 -04:00
parent efcdeb78c8
commit fd782219ef
No known key found for this signature in database
GPG key ID: 25C9B1A7F6798880
31 changed files with 314 additions and 158 deletions

View file

@ -1,16 +1,16 @@
import {Client, Message, Collection} from 'discord.js';
import {inject, injectable} from 'inversify';
import {TYPES} from './types';
import {Settings, Shortcut} from './models';
import container from './inversify.config';
import Command from './commands';
import debug from './utils/debug';
import NaturalLanguage from './services/natural-language-commands';
import handleGuildCreate from './events/guild-create';
import handleVoiceStateUpdate from './events/voice-state-update';
import errorMsg from './utils/error-msg';
import {isUserInVoice} from './utils/channels';
import Config from './services/config';
import {TYPES} from './types.js';
import {Settings, Shortcut} from './models/index.js';
import container from './inversify.config.js';
import Command from './commands/index.js';
import debug from './utils/debug.js';
import NaturalLanguage from './services/natural-language-commands.js';
import handleGuildCreate from './events/guild-create.js';
import handleVoiceStateUpdate from './events/voice-state-update.js';
import errorMsg from './utils/error-msg.js';
import {isUserInVoice} from './utils/channels.js';
import Config from './services/config.js';
@injectable()
export default class {

View file

@ -1,7 +1,7 @@
import {Message} from 'discord.js';
import {TYPES} from '../types';
import {inject, injectable} from 'inversify';
import PlayerManager from '../managers/player';
import {Message} from 'discord.js';
import {TYPES} from '../types.js';
import PlayerManager from '../managers/player.js';
import Command from '.';
@injectable()

View file

@ -1,7 +1,7 @@
import {TextChannel, Message, GuildChannel} from 'discord.js';
import {injectable} from 'inversify';
import {Settings} from '../models';
import errorMsg from '../utils/error-msg';
import {Settings} from '../models/index.js';
import errorMsg from '../utils/error-msg.js';
import Command from '.';
@injectable()

View file

@ -1,8 +1,8 @@
import {Message} from 'discord.js';
import {TYPES} from '../types';
import {TYPES} from '../types.js';
import {inject, injectable} from 'inversify';
import PlayerManager from '../managers/player';
import errorMsg from '../utils/error-msg';
import PlayerManager from '../managers/player.js';
import errorMsg from '../utils/error-msg.js';
import Command from '.';
@injectable()

View file

@ -1,9 +1,9 @@
import {Message, TextChannel} from 'discord.js';
import {TYPES} from '../types';
import {TYPES} from '../types.js';
import {inject, injectable} from 'inversify';
import PlayerManager from '../managers/player';
import LoadingMessage from '../utils/loading-message';
import errorMsg from '../utils/error-msg';
import PlayerManager from '../managers/player.js';
import LoadingMessage from '../utils/loading-message.js';
import errorMsg from '../utils/error-msg.js';
import Command from '.';
@injectable()

View file

@ -1,9 +1,9 @@
import {Message} from 'discord.js';
import {injectable} from 'inversify';
import Command from '.';
import {TYPES} from '../types';
import {Settings} from '../models';
import container from '../inversify.config';
import {TYPES} from '../types.js';
import {Settings} from '../models/index.js';
import container from '../inversify.config.js';
@injectable()
export default class implements Command {

View file

@ -1,9 +1,9 @@
import {Message} from 'discord.js';
import {TYPES} from '../types';
import {TYPES} from '../types.js';
import {inject, injectable} from 'inversify';
import PlayerManager from '../managers/player';
import {STATUS} from '../services/player';
import errorMsg from '../utils/error-msg';
import PlayerManager from '../managers/player.js';
import {STATUS} from '../services/player.js';
import errorMsg from '../utils/error-msg.js';
import Command from '.';
@injectable()

View file

@ -1,15 +1,15 @@
import {TextChannel, Message} from 'discord.js';
import {URL} from 'url';
import {Except} from 'type-fest';
import {TYPES} from '../types';
import {TYPES} from '../types.js';
import {inject, injectable} from 'inversify';
import {QueuedSong, STATUS} from '../services/player';
import PlayerManager from '../managers/player';
import {getMostPopularVoiceChannel, getMemberVoiceChannel} from '../utils/channels';
import LoadingMessage from '../utils/loading-message';
import errorMsg from '../utils/error-msg';
import {QueuedSong, STATUS} from '../services/player.js';
import PlayerManager from '../managers/player.js';
import {getMostPopularVoiceChannel, getMemberVoiceChannel} from '../utils/channels.js';
import LoadingMessage from '../utils/loading-message.js';
import errorMsg from '../utils/error-msg.js';
import Command from '.';
import GetSongs from '../services/get-songs';
import GetSongs from '../services/get-songs.js';
@injectable()
export default class implements Command {
@ -124,6 +124,7 @@ export default class implements Command {
if (song) {
newSongs.push(song);
} else {
console.log(_);
await res.stop(errorMsg('that doesn\'t exist'));
return;
}

View file

@ -1,13 +1,13 @@
import {Message, MessageEmbed} from 'discord.js';
import {TYPES} from '../types';
import {inject, injectable} from 'inversify';
import PlayerManager from '../managers/player';
import {STATUS} from '../services/player';
import Command from '.';
import getProgressBar from '../utils/get-progress-bar';
import errorMsg from '../utils/error-msg';
import {prettyTime} from '../utils/time';
import getYouTubeID from 'get-youtube-id';
import {inject, injectable} from 'inversify';
import {TYPES} from '../types.js';
import PlayerManager from '../managers/player.js';
import {STATUS} from '../services/player.js';
import Command from '.';
import getProgressBar from '../utils/get-progress-bar.js';
import errorMsg from '../utils/error-msg.js';
import {prettyTime} from '../utils/time.js';
const PAGE_SIZE = 10;

View file

@ -1,11 +1,11 @@
import {Message, TextChannel} from 'discord.js';
import {TYPES} from '../types';
import {TYPES} from '../types.js';
import {inject, injectable} from 'inversify';
import PlayerManager from '../managers/player';
import LoadingMessage from '../utils/loading-message';
import errorMsg from '../utils/error-msg';
import PlayerManager from '../managers/player.js';
import LoadingMessage from '../utils/loading-message.js';
import errorMsg from '../utils/error-msg.js';
import Command from '.';
import {parseTime} from '../utils/time';
import {parseTime} from '../utils/time.js';
@injectable()
export default class implements Command {

View file

@ -1,7 +1,7 @@
import {Message} from 'discord.js';
import {injectable} from 'inversify';
import {Shortcut, Settings} from '../models';
import errorMsg from '../utils/error-msg';
import {Shortcut, Settings} from '../models/index.js';
import errorMsg from '../utils/error-msg.js';
import Command from '.';
@injectable()

View file

@ -1,8 +1,8 @@
import {Message} from 'discord.js';
import {TYPES} from '../types';
import {TYPES} from '../types.js';
import {inject, injectable} from 'inversify';
import PlayerManager from '../managers/player';
import errorMsg from '../utils/error-msg';
import PlayerManager from '../managers/player.js';
import errorMsg from '../utils/error-msg.js';
import Command from '.';
@injectable()

View file

@ -1,10 +1,10 @@
import {Message, TextChannel} from 'discord.js';
import {TYPES} from '../types';
import {TYPES} from '../types.js';
import {inject, injectable} from 'inversify';
import PlayerManager from '../managers/player';
import PlayerManager from '../managers/player.js';
import Command from '.';
import LoadingMessage from '../utils/loading-message';
import errorMsg from '../utils/error-msg';
import LoadingMessage from '../utils/loading-message.js';
import errorMsg from '../utils/error-msg.js';
@injectable()
export default class implements Command {

View file

@ -1,8 +1,8 @@
import {Message} from 'discord.js';
import {TYPES} from '../types';
import {TYPES} from '../types.js';
import {inject, injectable} from 'inversify';
import PlayerManager from '../managers/player';
import errorMsg from '../utils/error-msg';
import PlayerManager from '../managers/player.js';
import errorMsg from '../utils/error-msg.js';
import Command from '.';
@injectable()

View file

@ -1,8 +1,8 @@
import {Guild, TextChannel, Message} from 'discord.js';
import emoji from 'node-emoji';
import pEvent from 'p-event';
import {Settings} from '../models';
import {chunk} from '../utils/arrays';
import {Settings} from '../models/index.js';
import {chunk} from '../utils/arrays.js';
const DEFAULT_PREFIX = '!';

View file

@ -1,8 +1,8 @@
import {VoiceState} from 'discord.js';
import container from '../inversify.config';
import {TYPES} from '../types';
import PlayerManager from '../managers/player';
import {getSizeWithoutBots} from '../utils/channels';
import container from '../inversify.config.js';
import {TYPES} from '../types.js';
import PlayerManager from '../managers/player.js';
import {getSizeWithoutBots} from '../utils/channels.js';
export default (oldState: VoiceState, _: VoiceState): void => {
const playerManager = container.get<PlayerManager>(TYPES.Managers.Player);

View file

@ -1,10 +1,10 @@
import makeDir from 'make-dir';
import path from 'path';
import container from './inversify.config';
import {TYPES} from './types';
import Bot from './bot';
import {sequelize} from './utils/db';
import Config from './services/config';
import container from './inversify.config.js';
import {TYPES} from './types.js';
import Bot from './bot.js';
import {sequelize} from './utils/db.js';
import Config from './services/config.js';
const bot = container.get<Bot>(TYPES.Bot);
@ -16,7 +16,7 @@ const bot = container.get<Bot>(TYPES.Bot);
await makeDir(config.CACHE_DIR);
await makeDir(path.join(config.CACHE_DIR, 'tmp'));
await sequelize.sync({});
await sequelize.sync({alter: true});
await bot.listen();
})();

View file

@ -1,33 +1,34 @@
import 'reflect-metadata';
import {Container} from 'inversify';
import {TYPES} from './types';
import Bot from './bot';
import {TYPES} from './types.js';
import Bot from './bot.js';
import {Client} from 'discord.js';
import ConfigProvider from './services/config';
import ConfigProvider from './services/config.js';
// Managers
import PlayerManager from './managers/player';
import PlayerManager from './managers/player.js';
// Helpers
import GetSongs from './services/get-songs';
import NaturalLanguage from './services/natural-language-commands';
import GetSongs from './services/get-songs.js';
import NaturalLanguage from './services/natural-language-commands.js';
// Comands
import Command from './commands';
import Clear from './commands/clear';
import Config from './commands/config';
import Disconnect from './commands/disconnect';
import ForwardSeek from './commands/fseek';
import Help from './commands/help';
import Pause from './commands/pause';
import Play from './commands/play';
import QueueCommad from './commands/queue';
import Seek from './commands/seek';
import Shortcuts from './commands/shortcuts';
import Shuffle from './commands/shuffle';
import Skip from './commands/skip';
import Unskip from './commands/unskip';
import ThirdParty from './services/third-party';
import Clear from './commands/clear.js';
import Config from './commands/config.js';
import Disconnect from './commands/disconnect.js';
import ForwardSeek from './commands/fseek.js';
import Help from './commands/help.js';
import Pause from './commands/pause.js';
import Play from './commands/play.js';
import QueueCommad from './commands/queue.js';
import Seek from './commands/seek.js';
import Shortcuts from './commands/shortcuts.js';
import Shuffle from './commands/shuffle.js';
import Skip from './commands/skip.js';
import Unskip from './commands/unskip.js';
import ThirdParty from './services/third-party.js';
import CacheProvider from './services/cache.js';
let container = new Container();
@ -67,4 +68,6 @@ container.bind(TYPES.Config).toConstantValue(new ConfigProvider());
// Static libraries
container.bind(TYPES.ThirdParty).to(ThirdParty);
container.bind(TYPES.Cache).to(CacheProvider);
export default container;

View file

@ -1,8 +1,8 @@
import {inject, injectable} from 'inversify';
import {TYPES} from '../types';
import Player from '../services/player';
import {Client} from 'discord.js';
import Config from '../services/config';
import {TYPES} from '../types.js';
import Player from '../services/player.js';
import Config from '../services/config.js';
@injectable()
export default class {

15
src/models/cache.ts Normal file
View file

@ -0,0 +1,15 @@
import {Table, Column, PrimaryKey, Model} from 'sequelize-typescript';
import sequelize from 'sequelize';
@Table
export default class Cache extends Model<Cache> {
@PrimaryKey
@Column
key!: string;
@Column(sequelize.TEXT)
value!: string;
@Column
expiresAt!: Date;
}

View file

@ -1,7 +1,9 @@
import Settings from './settings';
import Shortcut from './shortcut';
import Cache from './cache.js';
import Settings from './settings.js';
import Shortcut from './shortcut.js';
export {
Cache,
Settings,
Shortcut
};

52
src/services/cache.ts Normal file
View file

@ -0,0 +1,52 @@
import {injectable} from 'inversify';
import {Cache} from '../models/index.js';
import debug from '../utils/debug.js';
type Seconds = number;
type Options = {
expiresIn: Seconds;
key?: string;
};
const futureTimeToDate = (time: Seconds) => new Date(new Date().getTime() + (time * 1000));
@injectable()
export default class CacheProvider {
async wrap<T extends [...any[], Options], F>(func: (...options: any) => Promise<F>, ...options: T): Promise<F> {
if (options.length === 0) {
throw new Error('Missing cache options');
}
const functionArgs = options.slice(0, options.length - 1);
const {
key = JSON.stringify(functionArgs),
expiresIn
} = options[options.length - 1] as Options;
const cachedResult = await Cache.findByPk(key);
if (cachedResult) {
if (new Date() < cachedResult.expiresAt) {
debug(`Cache hit: ${key}`);
return JSON.parse(cachedResult.value);
}
await cachedResult.destroy();
}
debug(`Cache miss: ${key}`);
const result = await func(...options as any[]);
// Save result
await Cache.upsert({
key,
value: JSON.stringify(result),
expiresAt: futureTimeToDate(expiresIn)
});
return result;
}
}

View file

@ -2,37 +2,73 @@ import {URL} from 'url';
import {inject, injectable} from 'inversify';
import {toSeconds, parse} from 'iso8601-duration';
import got from 'got';
import ytsr, {Video} from 'ytsr';
import spotifyURI from 'spotify-uri';
import Spotify from 'spotify-web-api-node';
import YouTube, {YoutubePlaylistItem} from 'youtube.ts';
import pLimit from 'p-limit';
import PQueue from 'p-queue';
import shuffle from 'array-shuffle';
import {Except} from 'type-fest';
import {QueuedSong, QueuedPlaylist} from '../services/player';
import {TYPES} from '../types';
import {cleanUrl} from '../utils/url';
import ThirdParty from './third-party';
import Config from './config';
import {QueuedSong, QueuedPlaylist} from '../services/player.js';
import {TYPES} from '../types.js';
import {cleanUrl} from '../utils/url.js';
import ThirdParty from './third-party.js';
import Config from './config.js';
import CacheProvider from './cache.js';
type QueuedSongWithoutChannel = Except<QueuedSong, 'addedInChannelId'>;
const ONE_HOUR_IN_SECONDS = 60 * 60;
const ONE_MINUTE_IN_SECONDS = 1 * 60;
@injectable()
export default class {
private readonly youtube: YouTube;
private readonly youtubeKey: string;
private readonly spotify: Spotify;
private readonly cache: CacheProvider;
constructor(@inject(TYPES.ThirdParty) thirdParty: ThirdParty, @inject(TYPES.Config) config: Config) {
private readonly ytsrQueue: PQueue;
constructor(
@inject(TYPES.ThirdParty) thirdParty: ThirdParty,
@inject(TYPES.Config) config: Config,
@inject(TYPES.Cache) cache: CacheProvider) {
this.youtube = thirdParty.youtube;
this.youtubeKey = config.YOUTUBE_API_KEY;
this.spotify = thirdParty.spotify;
this.cache = cache;
this.ytsrQueue = new PQueue({concurrency: 4});
}
async youtubeVideoSearch(query: string): Promise<QueuedSongWithoutChannel|null> {
try {
const {items: [video]} = await this.youtube.videos.search({q: query, maxResults: 1, type: 'video'});
const {items} = await this.ytsrQueue.add(async () => this.cache.wrap(
ytsr,
query,
{
limit: 10
},
{
expiresIn: ONE_HOUR_IN_SECONDS
}
));
return await this.youtubeVideo(video.id.videoId);
let firstVideo: Video | undefined;
for (const item of items) {
if (item.type === 'video') {
firstVideo = item;
break;
}
}
if (!firstVideo) {
throw new Error('No video found.');
}
return await this.youtubeVideo(firstVideo.id);
} catch (_: unknown) {
return null;
}
@ -40,7 +76,13 @@ export default class {
async youtubeVideo(url: string): Promise<QueuedSongWithoutChannel|null> {
try {
const videoDetails = await this.youtube.videos.get(cleanUrl(url));
const videoDetails = await this.cache.wrap(
this.youtube.videos.get,
cleanUrl(url),
{
expiresIn: ONE_HOUR_IN_SECONDS
}
);
return {
title: videoDetails.snippet.title,
@ -57,7 +99,13 @@ export default class {
async youtubePlaylist(listId: string): Promise<QueuedSongWithoutChannel[]> {
// YouTube playlist
const playlist = await this.youtube.playlists.get(listId);
const playlist = await this.cache.wrap(
this.youtube.playlists.get,
listId,
{
expiresIn: ONE_MINUTE_IN_SECONDS
}
);
interface VideoDetailsResponse {
id: string;
@ -75,7 +123,14 @@ export default class {
while (playlistVideos.length !== playlist.contentDetails.itemCount) {
// eslint-disable-next-line no-await-in-loop
const {items, nextPageToken} = await this.youtube.playlists.items(listId, {maxResults: '50', pageToken: nextToken});
const {items, nextPageToken} = await this.cache.wrap(
this.youtube.playlists.items,
listId,
{maxResults: '50', pageToken: nextToken},
{
expiresIn: ONE_MINUTE_IN_SECONDS
}
);
nextToken = nextPageToken;
@ -84,11 +139,24 @@ export default class {
// Start fetching extra details about videos
videoDetailsPromises.push((async () => {
// Unfortunately, package doesn't provide a method for this
const {items: videoDetailItems}: {items: VideoDetailsResponse[]} = await got('https://www.googleapis.com/youtube/v3/videos', {searchParams: {
part: 'contentDetails',
id: items.map(item => item.contentDetails.videoId).join(','),
key: this.youtubeKey
}}).json();
const {items: videoDetailItems} = await this.cache.wrap(
() => {
return got(
'https://www.googleapis.com/youtube/v3/videos',
{
searchParams: {
part: 'contentDetails',
id: items.map(item => item.contentDetails.videoId).join(','),
key: this.youtubeKey,
responseType: 'json'
}
}
).json();
},
{
expiresIn: ONE_MINUTE_IN_SECONDS
}
);
videoDetails.push(...videoDetailItems);
})());
@ -193,9 +261,7 @@ export default class {
tracks = shuffled.slice(0, 50);
}
// Limit concurrency so hopefully we don't get banned for searching
const limit = pLimit(5);
let songs = await Promise.all(tracks.map(async track => limit(async () => this.spotifyToYouTube(track, playlist))));
let songs = await Promise.all(tracks.map(async track => this.spotifyToYouTube(track, playlist)));
let nSongsNotFound = 0;
@ -215,14 +281,7 @@ export default class {
private async spotifyToYouTube(track: SpotifyApi.TrackObjectSimplified, _: QueuedPlaylist | null): Promise<QueuedSongWithoutChannel | null> {
try {
const {items} = await this.youtube.videos.search({q: `"${track.name}" "${track.artists[0].name}"`, maxResults: 10});
const videoResult = items[0];
if (!videoResult) {
throw new Error('No video found for query.');
}
return await this.youtubeVideo(videoResult.id.videoId);
return await this.youtubeVideoSearch(`"${track.name}" "${track.artists[0].name}"`);
} catch (_: unknown) {
return null;
}

View file

@ -1,9 +1,9 @@
import {inject, injectable} from 'inversify';
import {Message, Guild, GuildMember} from 'discord.js';
import {TYPES} from '../types';
import PlayerManager from '../managers/player';
import {QueuedSong} from '../services/player';
import {getMostPopularVoiceChannel, getMemberVoiceChannel} from '../utils/channels';
import {TYPES} from '../types.js';
import PlayerManager from '../managers/player.js';
import {QueuedSong} from '../services/player.js';
import {getMostPopularVoiceChannel, getMemberVoiceChannel} from '../utils/channels.js';
@injectable()
export default class {

View file

@ -7,7 +7,7 @@ import ytdl from 'ytdl-core';
import {WriteStream} from 'fs-capacitor';
import ffmpeg from 'fluent-ffmpeg';
import shuffle from 'array-shuffle';
import errorMsg from '../utils/error-msg';
import errorMsg from '../utils/error-msg.js';
export interface QueuedPlaylist {
title: string;

View file

@ -1,8 +1,8 @@
import {inject, injectable} from 'inversify';
import SpotifyWebApi from 'spotify-web-api-node';
import Youtube from 'youtube.ts';
import {TYPES} from '../types';
import Config from './config';
import Youtube from 'youtube.ts/dist/youtube.js';
import {TYPES} from '../types.js';
import Config from './config.js';
@injectable()
export default class ThirdParty {
@ -12,7 +12,8 @@ export default class ThirdParty {
private spotifyTokenTimerId?: NodeJS.Timeout;
constructor(@inject(TYPES.Config) config: Config) {
this.youtube = new Youtube(config.YOUTUBE_API_KEY);
// Library is transpiled incorrectly
this.youtube = new ((Youtube as any).default)(config.YOUTUBE_API_KEY);
this.spotify = new SpotifyWebApi({
clientId: config.SPOTIFY_CLIENT_ID,
clientSecret: config.SPOTIFY_CLIENT_SECRET

View file

@ -1,5 +1,6 @@
export const TYPES = {
Bot: Symbol('Bot'),
Cache: Symbol('Cache'),
Client: Symbol('Client'),
Config: Symbol('Config'),
Command: Symbol('Command'),

View file

@ -1,12 +1,12 @@
import {Sequelize} from 'sequelize-typescript';
import path from 'path';
import {DATA_DIR} from '../services/config';
import {Settings, Shortcut} from '../models';
import {DATA_DIR} from '../services/config.js';
import {Cache, Settings, Shortcut} from '../models/index.js';
export const sequelize = new Sequelize({
dialect: 'sqlite',
database: 'muse',
storage: path.join(DATA_DIR, 'db.sqlite'),
models: [Settings, Shortcut],
models: [Cache, Settings, Shortcut],
logging: false
});