"use strict";
Object.defineProperty(exports, "__esModule", { value: true });
exports.JobQueue = exports.jobTypes = void 0;
const tslib_1 = require("tslib");
const bullmq_1 = require("bullmq");
const jobs_1 = require("../../helpers/custom-validators/jobs");
const config_1 = require("../../initializers/config");
const video_redundancy_1 = require("./handlers/video-redundancy");
const core_utils_1 = require("../../../shared/core-utils");
const logger_1 = require("../../helpers/logger");
const constants_1 = require("../../initializers/constants");
const hooks_1 = require("../plugins/hooks");
const redis_1 = require("../redis");
const activitypub_cleaner_1 = require("./handlers/activitypub-cleaner");
const activitypub_follow_1 = require("./handlers/activitypub-follow");
const activitypub_http_broadcast_1 = require("./handlers/activitypub-http-broadcast");
const activitypub_http_fetcher_1 = require("./handlers/activitypub-http-fetcher");
const activitypub_http_unicast_1 = require("./handlers/activitypub-http-unicast");
const activitypub_refresher_1 = require("./handlers/activitypub-refresher");
const actor_keys_1 = require("./handlers/actor-keys");
const after_video_channel_import_1 = require("./handlers/after-video-channel-import");
const email_1 = require("./handlers/email");
const federate_video_1 = require("./handlers/federate-video");
const manage_video_torrent_1 = require("./handlers/manage-video-torrent");
const move_to_object_storage_1 = require("./handlers/move-to-object-storage");
const notify_1 = require("./handlers/notify");
const video_channel_import_1 = require("./handlers/video-channel-import");
const video_file_import_1 = require("./handlers/video-file-import");
const video_import_1 = require("./handlers/video-import");
const video_live_ending_1 = require("./handlers/video-live-ending");
const video_studio_edition_1 = require("./handlers/video-studio-edition");
const video_transcoding_1 = require("./handlers/video-transcoding");
const video_views_stats_1 = require("./handlers/video-views-stats");
const core_utils_2 = require("../../helpers/core-utils");
const handlers = {
    'activitypub-http-broadcast': activitypub_http_broadcast_1.processActivityPubHttpSequentialBroadcast,
    'activitypub-http-broadcast-parallel': activitypub_http_broadcast_1.processActivityPubParallelHttpBroadcast,
    'activitypub-http-unicast': activitypub_http_unicast_1.processActivityPubHttpUnicast,
    'activitypub-http-fetcher': activitypub_http_fetcher_1.processActivityPubHttpFetcher,
    'activitypub-cleaner': activitypub_cleaner_1.processActivityPubCleaner,
    'activitypub-follow': activitypub_follow_1.processActivityPubFollow,
    'video-file-import': video_file_import_1.processVideoFileImport,
    'video-transcoding': video_transcoding_1.processVideoTranscoding,
    'email': email_1.processEmail,
    'video-import': video_import_1.processVideoImport,
    'videos-views-stats': video_views_stats_1.processVideosViewsStats,
    'activitypub-refresher': activitypub_refresher_1.refreshAPObject,
    'video-live-ending': video_live_ending_1.processVideoLiveEnding,
    'actor-keys': actor_keys_1.processActorKeys,
    'video-redundancy': video_redundancy_1.processVideoRedundancy,
    'move-to-object-storage': move_to_object_storage_1.processMoveToObjectStorage,
    'manage-video-torrent': manage_video_torrent_1.processManageVideoTorrent,
    'video-studio-edition': video_studio_edition_1.processVideoStudioEdition,
    'video-channel-import': video_channel_import_1.processVideoChannelImport,
    'after-video-channel-import': after_video_channel_import_1.processAfterVideoChannelImport,
    'notify': notify_1.processNotify,
    'federate-video': federate_video_1.processFederateVideo
};
const errorHandlers = {
    'move-to-object-storage': move_to_object_storage_1.onMoveToObjectStorageFailure
};
const jobTypes = [
    'activitypub-follow',
    'activitypub-http-broadcast',
    'activitypub-http-broadcast-parallel',
    'activitypub-http-fetcher',
    'activitypub-http-unicast',
    'activitypub-cleaner',
    'email',
    'video-transcoding',
    'video-file-import',
    'video-import',
    'videos-views-stats',
    'activitypub-refresher',
    'video-redundancy',
    'actor-keys',
    'video-live-ending',
    'move-to-object-storage',
    'manage-video-torrent',
    'video-studio-edition',
    'video-channel-import',
    'after-video-channel-import',
    'notify',
    'federate-video'
];
exports.jobTypes = jobTypes;
const silentFailure = new Set(['activitypub-http-unicast']);
class JobQueue {
    constructor() {
        this.workers = {};
        this.queues = {};
        this.queueSchedulers = {};
        this.queueEvents = {};
        this.initialized = false;
    }
    init() {
        if (this.initialized === true)
            return;
        this.initialized = true;
        this.jobRedisPrefix = 'bull-' + constants_1.WEBSERVER.HOST;
        for (const handlerName of Object.keys(handlers)) {
            this.buildWorker(handlerName);
            this.buildQueue(handlerName);
            this.buildQueueScheduler(handlerName);
            this.buildQueueEvent(handlerName);
        }
        this.flowProducer = new bullmq_1.FlowProducer({
            connection: redis_1.Redis.getRedisClientOptions('FlowProducer'),
            prefix: this.jobRedisPrefix
        });
        this.flowProducer.on('error', err => { logger_1.logger.error('Error in flow producer', { err }); });
        this.addRepeatableJobs();
    }
    buildWorker(handlerName) {
        const workerOptions = {
            autorun: false,
            concurrency: this.getJobConcurrency(handlerName),
            prefix: this.jobRedisPrefix,
            connection: redis_1.Redis.getRedisClientOptions('Worker')
        };
        const handler = function (job) {
            const timeout = constants_1.JOB_TTL[handlerName];
            const p = handlers[handlerName](job);
            if (!timeout)
                return p;
            return (0, core_utils_1.timeoutPromise)(p, timeout);
        };
        const processor = (jobArg) => tslib_1.__awaiter(this, void 0, void 0, function* () {
            const job = yield hooks_1.Hooks.wrapObject(jobArg, 'filter:job-queue.process.params', { type: handlerName });
            return hooks_1.Hooks.wrapPromiseFun(handler, job, 'filter:job-queue.process.result');
        });
        const worker = new bullmq_1.Worker(handlerName, processor, workerOptions);
        worker.on('failed', (job, err) => {
            const logLevel = silentFailure.has(handlerName)
                ? 'debug'
                : 'error';
            logger_1.logger.log(logLevel, 'Cannot execute job %s in queue %s.', job.id, handlerName, { payload: job.data, err });
            if (errorHandlers[job.name]) {
                errorHandlers[job.name](job, err)
                    .catch(err => logger_1.logger.error('Cannot run error handler for job failure %d in queue %s.', job.id, handlerName, { err }));
            }
        });
        worker.on('error', err => { logger_1.logger.error('Error in job worker %s.', handlerName, { err }); });
        this.workers[handlerName] = worker;
    }
    buildQueue(handlerName) {
        const queueOptions = {
            connection: redis_1.Redis.getRedisClientOptions('Queue'),
            prefix: this.jobRedisPrefix
        };
        const queue = new bullmq_1.Queue(handlerName, queueOptions);
        queue.on('error', err => { logger_1.logger.error('Error in job queue %s.', handlerName, { err }); });
        this.queues[handlerName] = queue;
    }
    buildQueueScheduler(handlerName) {
        const queueSchedulerOptions = {
            autorun: false,
            connection: redis_1.Redis.getRedisClientOptions('QueueScheduler'),
            prefix: this.jobRedisPrefix,
            maxStalledCount: 10
        };
        const queueScheduler = new bullmq_1.QueueScheduler(handlerName, queueSchedulerOptions);
        queueScheduler.on('error', err => { logger_1.logger.error('Error in job queue scheduler %s.', handlerName, { err }); });
        this.queueSchedulers[handlerName] = queueScheduler;
    }
    buildQueueEvent(handlerName) {
        const queueEventsOptions = {
            autorun: false,
            connection: redis_1.Redis.getRedisClientOptions('QueueEvent'),
            prefix: this.jobRedisPrefix
        };
        const queueEvents = new bullmq_1.QueueEvents(handlerName, queueEventsOptions);
        queueEvents.on('error', err => { logger_1.logger.error('Error in job queue events %s.', handlerName, { err }); });
        this.queueEvents[handlerName] = queueEvents;
    }
    terminate() {
        return tslib_1.__awaiter(this, void 0, void 0, function* () {
            const promises = Object.keys(this.workers)
                .map(handlerName => {
                const worker = this.workers[handlerName];
                const queue = this.queues[handlerName];
                const queueScheduler = this.queueSchedulers[handlerName];
                const queueEvent = this.queueEvents[handlerName];
                return Promise.all([
                    worker.close(false),
                    queue.close(),
                    queueScheduler.close(),
                    queueEvent.close()
                ]);
            });
            return Promise.all(promises);
        });
    }
    start() {
        const promises = Object.keys(this.workers)
            .map(handlerName => {
            const worker = this.workers[handlerName];
            const queueScheduler = this.queueSchedulers[handlerName];
            const queueEvent = this.queueEvents[handlerName];
            return Promise.all([
                worker.run(),
                queueScheduler.run(),
                queueEvent.run()
            ]);
        });
        return Promise.all(promises);
    }
    pause() {
        return tslib_1.__awaiter(this, void 0, void 0, function* () {
            for (const handlerName of Object.keys(this.workers)) {
                const worker = this.workers[handlerName];
                yield worker.pause();
            }
        });
    }
    resume() {
        for (const handlerName of Object.keys(this.workers)) {
            const worker = this.workers[handlerName];
            worker.resume();
        }
    }
    createJobAsync(options) {
        this.createJob(options)
            .catch(err => logger_1.logger.error('Cannot create job.', { err, options }));
    }
    createJob(options) {
        const queue = this.queues[options.type];
        if (queue === undefined) {
            logger_1.logger.error('Unknown queue %s: cannot create job.', options.type);
            return;
        }
        const jobOptions = this.buildJobOptions(options.type, (0, core_utils_1.pick)(options, ['priority', 'delay']));
        return queue.add('job', options.payload, jobOptions);
    }
    createSequentialJobFlow(...jobs) {
        let lastJob;
        for (const job of jobs) {
            if (!job)
                continue;
            lastJob = Object.assign(Object.assign({}, this.buildJobFlowOption(job)), { children: lastJob
                    ? [lastJob]
                    : [] });
        }
        return this.flowProducer.add(lastJob);
    }
    createJobWithChildren(parent, children) {
        return this.flowProducer.add(Object.assign(Object.assign({}, this.buildJobFlowOption(parent)), { children: children.map(c => this.buildJobFlowOption(c)) }));
    }
    buildJobFlowOption(job) {
        return {
            name: 'job',
            data: job.payload,
            queueName: job.type,
            opts: this.buildJobOptions(job.type, (0, core_utils_1.pick)(job, ['priority', 'delay']))
        };
    }
    buildJobOptions(type, options = {}) {
        return Object.assign({ backoff: { delay: 60 * 1000, type: 'exponential' }, attempts: constants_1.JOB_ATTEMPTS[type], priority: options.priority, delay: options.delay }, this.buildJobRemovalOptions(type));
    }
    listForApi(options) {
        return tslib_1.__awaiter(this, void 0, void 0, function* () {
            const { state, start, count, asc, jobType } = options;
            const states = this.buildStateFilter(state);
            const filteredJobTypes = this.buildTypeFilter(jobType);
            let results = [];
            for (const jobType of filteredJobTypes) {
                const queue = this.queues[jobType];
                if (queue === undefined) {
                    logger_1.logger.error('Unknown queue %s to list jobs.', jobType);
                    continue;
                }
                const jobs = yield queue.getJobs(states, 0, start + count, asc);
                results = results.concat(jobs);
            }
            results.sort((j1, j2) => {
                if (j1.timestamp < j2.timestamp)
                    return -1;
                else if (j1.timestamp === j2.timestamp)
                    return 0;
                return 1;
            });
            if (asc === false)
                results.reverse();
            return results.slice(start, start + count);
        });
    }
    count(state, jobType) {
        return tslib_1.__awaiter(this, void 0, void 0, function* () {
            const states = state ? [state] : jobs_1.jobStates;
            const filteredJobTypes = this.buildTypeFilter(jobType);
            let total = 0;
            for (const type of filteredJobTypes) {
                const queue = this.queues[type];
                if (queue === undefined) {
                    logger_1.logger.error('Unknown queue %s to count jobs.', type);
                    continue;
                }
                const counts = yield queue.getJobCounts();
                for (const s of states) {
                    total += counts[s];
                }
            }
            return total;
        });
    }
    buildStateFilter(state) {
        if (!state)
            return jobs_1.jobStates;
        const states = [state];
        if (state === 'waiting')
            states.push('waiting-children');
        return states;
    }
    buildTypeFilter(jobType) {
        if (!jobType)
            return jobTypes;
        return jobTypes.filter(t => t === jobType);
    }
    getStats() {
        return tslib_1.__awaiter(this, void 0, void 0, function* () {
            const promises = jobTypes.map((t) => tslib_1.__awaiter(this, void 0, void 0, function* () { return ({ jobType: t, counts: yield this.queues[t].getJobCounts() }); }));
            return Promise.all(promises);
        });
    }
    removeOldJobs() {
        return tslib_1.__awaiter(this, void 0, void 0, function* () {
            for (const key of Object.keys(this.queues)) {
                const queue = this.queues[key];
                yield queue.clean((0, core_utils_2.parseDurationToMs)('7 days'), 1000, 'completed');
                yield queue.clean((0, core_utils_2.parseDurationToMs)('7 days'), 1000, 'failed');
            }
        });
    }
    addRepeatableJobs() {
        this.queues['videos-views-stats'].add('job', {}, Object.assign({ repeat: constants_1.REPEAT_JOBS['videos-views-stats'] }, this.buildJobRemovalOptions('videos-views-stats'))).catch(err => logger_1.logger.error('Cannot add repeatable job.', { err }));
        if (config_1.CONFIG.FEDERATION.VIDEOS.CLEANUP_REMOTE_INTERACTIONS) {
            this.queues['activitypub-cleaner'].add('job', {}, Object.assign({ repeat: constants_1.REPEAT_JOBS['activitypub-cleaner'] }, this.buildJobRemovalOptions('activitypub-cleaner'))).catch(err => logger_1.logger.error('Cannot add repeatable job.', { err }));
        }
    }
    getJobConcurrency(jobType) {
        if (jobType === 'video-transcoding')
            return config_1.CONFIG.TRANSCODING.CONCURRENCY;
        if (jobType === 'video-import')
            return config_1.CONFIG.IMPORT.VIDEOS.CONCURRENCY;
        return constants_1.JOB_CONCURRENCY[jobType];
    }
    buildJobRemovalOptions(queueName) {
        return {
            removeOnComplete: {
                age: (constants_1.JOB_REMOVAL_OPTIONS.SUCCESS[queueName] || constants_1.JOB_REMOVAL_OPTIONS.SUCCESS.DEFAULT) / 1000,
                count: constants_1.JOB_REMOVAL_OPTIONS.COUNT
            },
            removeOnFail: {
                age: (constants_1.JOB_REMOVAL_OPTIONS.FAILURE[queueName] || constants_1.JOB_REMOVAL_OPTIONS.FAILURE.DEFAULT) / 1000,
                count: constants_1.JOB_REMOVAL_OPTIONS.COUNT / 1000
            }
        };
    }
    static get Instance() {
        return this.instance || (this.instance = new this());
    }
}
exports.JobQueue = JobQueue;
//# sourceMappingURL=job-queue.js.map