Skip to content

Commit e556583

Browse files
ChocobozzzChocobozzz
committed
Improve redundancy: add 'min_lifetime' configuration
1 parent d1a63fc commit e556583

27 files changed

+627
-278
lines changed

‎config/default.yaml

+7-1
Original file line numberDiff line numberDiff line change
@@ -75,14 +75,20 @@ redundancy:
7575
strategies:
7676
# -
7777
# size: '10GB'
78+
# # Minimum time the video must remain in the cache. Only accept values > 10 hours (to not overload remote instances)
79+
# min_lifetime: '48 hours'
7880
# strategy: 'most-views' # Cache videos that have the most views
7981
# -
8082
# size: '10GB'
83+
# # Minimum time the video must remain in the cache. Only accept values > 10 hours (to not overload remote instances)
84+
# min_lifetime: '48 hours'
8185
# strategy: 'trending' # Cache trending videos
8286
# -
8387
# size: '10GB'
88+
# # Minimum time the video must remain in the cache. Only accept values > 10 hours (to not overload remote instances)
89+
# min_lifetime: '48 hours'
8490
# strategy: 'recently-added' # Cache recently added videos
85-
# minViews: 10 # Having at least x views
91+
# min_views: 10 # Having at least x views
8692

8793
cache:
8894
previews:

‎config/production.yaml.example

+7-1
Original file line numberDiff line numberDiff line change
@@ -76,14 +76,20 @@ redundancy:
7676
strategies:
7777
# -
7878
# size: '10GB'
79+
# # Minimum time the video must remain in the cache. Only accept values > 10 hours (to not overload remote instances)
80+
# min_lifetime: '48 hours'
7981
# strategy: 'most-views' # Cache videos that have the most views
8082
# -
8183
# size: '10GB'
84+
# # Minimum time the video must remain in the cache. Only accept values > 10 hours (to not overload remote instances)
85+
# min_lifetime: '48 hours'
8286
# strategy: 'trending' # Cache trending videos
8387
# -
8488
# size: '10GB'
89+
# # Minimum time the video must remain in the cache. Only accept values > 10 hours (to not overload remote instances)
90+
# min_lifetime: '48 hours'
8591
# strategy: 'recently-added' # Cache recently added videos
86-
# minViews: 10 # Having at least x views
92+
# min_views: 10 # Having at least x views
8793

8894
###############################################################################
8995
#

‎config/test.yaml

+5-2
Original file line numberDiff line numberDiff line change
@@ -23,18 +23,21 @@ log:
2323

2424
redundancy:
2525
videos:
26-
check_interval: '5 seconds'
26+
check_interval: '10 minutes'
2727
strategies:
2828
-
2929
size: '10MB'
30+
min_lifetime: '10 minutes'
3031
strategy: 'most-views'
3132
-
3233
size: '10MB'
34+
min_lifetime: '10 minutes'
3335
strategy: 'trending'
3436
-
3537
size: '10MB'
38+
min_lifetime: '10 minutes'
3639
strategy: 'recently-added'
37-
minViews: 1
40+
min_views: 1
3841

3942
cache:
4043
previews:

‎server.ts

+6-6
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,4 @@
11
// FIXME: https://github.com/nodejs/node/pull/16853
2-
import { VideosCaptionCache } from './server/lib/cache/videos-caption-cache'
3-
42
require('tls').DEFAULT_ECDH_CURVE = 'auto'
53

64
import { isTestInstance } from './server/helpers/core-utils'
@@ -17,15 +15,15 @@ import * as cors from 'cors'
1715
import * as cookieParser from 'cookie-parser'
1816
import * as helmet from 'helmet'
1917
import * as useragent from 'useragent'
20-
import * as anonymise from 'ip-anonymize'
18+
import * as anonymize from 'ip-anonymize'
2119

2220
process.title = 'peertube'
2321

2422
// Create our main app
2523
const app = express()
2624

2725
// ----------- Core checker -----------
28-
import { checkMissedConfig, checkFFmpeg, checkConfig, checkActivityPubUrls } from './server/initializers/checker'
26+
import { checkMissedConfig, checkFFmpeg } from './server/initializers/checker-before-init'
2927

3028
// Do not use barrels because we don't want to load all modules here (we need to initialize database first)
3129
import { logger } from './server/helpers/logger'
@@ -43,6 +41,8 @@ checkFFmpeg(CONFIG)
4341
process.exit(-1)
4442
})
4543

44+
import { checkConfig, checkActivityPubUrls } from './server/initializers/checker-after-init'
45+
4646
const errorMessage = checkConfig()
4747
if (errorMessage !== null) {
4848
throw new Error(errorMessage)
@@ -76,7 +76,7 @@ migrate()
7676
import { installApplication } from './server/initializers'
7777
import { Emailer } from './server/lib/emailer'
7878
import { JobQueue } from './server/lib/job-queue'
79-
import { VideosPreviewCache } from './server/lib/cache'
79+
import { VideosPreviewCache, VideosCaptionCache } from './server/lib/cache'
8080
import {
8181
activityPubRouter,
8282
apiRouter,
@@ -111,7 +111,7 @@ if (isTestInstance()) {
111111
// For the logger
112112
morgan.token('remote-addr', req => {
113113
return (req.get('DNT') === '1') ?
114-
anonymise(req.ip || (req.connection && req.connection.remoteAddress) || undefined,
114+
anonymize(req.ip || (req.connection && req.connection.remoteAddress) || undefined,
115115
16, // bitmask for IPv4
116116
16 // bitmask for IPv6
117117
) :

‎server/helpers/ffmpeg-utils.ts

+1-1
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ import { VideoResolution } from '../../shared/models/videos'
44
import { CONFIG, FFMPEG_NICE, VIDEO_TRANSCODING_FPS } from '../initializers'
55
import { processImage } from './image-utils'
66
import { logger } from './logger'
7-
import { checkFFmpegEncoders } from '../initializers/checker'
7+
import { checkFFmpegEncoders } from '../initializers/checker-before-init'
88
import { remove } from 'fs-extra'
99

1010
function computeResolutionsToTranscode (videoFileHeight: number) {

‎server/initializers/checker-after-init.ts

+115
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,115 @@
1+
import * as config from 'config'
2+
import { promisify0, isProdInstance, parseDuration, isTestInstance } from '../helpers/core-utils'
3+
import { UserModel } from '../models/account/user'
4+
import { ApplicationModel } from '../models/application/application'
5+
import { OAuthClientModel } from '../models/oauth/oauth-client'
6+
import { parse } from 'url'
7+
import { CONFIG } from './constants'
8+
import { logger } from '../helpers/logger'
9+
import { getServerActor } from '../helpers/utils'
10+
import { RecentlyAddedStrategy, VideosRedundancy } from '../../shared/models/redundancy'
11+
import { isArray } from '../helpers/custom-validators/misc'
12+
import { uniq } from 'lodash'
13+
14+
async function checkActivityPubUrls () {
15+
const actor = await getServerActor()
16+
17+
const parsed = parse(actor.url)
18+
if (CONFIG.WEBSERVER.HOST !== parsed.host) {
19+
const NODE_ENV = config.util.getEnv('NODE_ENV')
20+
const NODE_CONFIG_DIR = config.util.getEnv('NODE_CONFIG_DIR')
21+
22+
logger.warn(
23+
'It seems PeerTube was started (and created some data) with another domain name. ' +
24+
'This means you will not be able to federate! ' +
25+
'Please use %s %s npm run update-host to fix this.',
26+
NODE_CONFIG_DIR ? `NODE_CONFIG_DIR=${NODE_CONFIG_DIR}` : '',
27+
NODE_ENV ? `NODE_ENV=${NODE_ENV}` : ''
28+
)
29+
}
30+
}
31+
32+
// Some checks on configuration files
33+
// Return an error message, or null if everything is okay
34+
function checkConfig () {
35+
const defaultNSFWPolicy = CONFIG.INSTANCE.DEFAULT_NSFW_POLICY
36+
37+
// NSFW policy
38+
{
39+
const available = [ 'do_not_list', 'blur', 'display' ]
40+
if (available.indexOf(defaultNSFWPolicy) === -1) {
41+
return 'NSFW policy setting should be ' + available.join(' or ') + ' instead of ' + defaultNSFWPolicy
42+
}
43+
}
44+
45+
// Redundancies
46+
const redundancyVideos = CONFIG.REDUNDANCY.VIDEOS.STRATEGIES
47+
if (isArray(redundancyVideos)) {
48+
const available = [ 'most-views', 'trending', 'recently-added' ]
49+
for (const r of redundancyVideos) {
50+
if (available.indexOf(r.strategy) === -1) {
51+
return 'Videos redundancy should have ' + available.join(' or ') + ' strategy instead of ' + r.strategy
52+
}
53+
54+
// Lifetime should not be < 10 hours
55+
if (!isTestInstance() && r.minLifetime < 1000 * 3600 * 10) {
56+
return 'Video redundancy minimum lifetime should be >= 10 hours for strategy ' + r.strategy
57+
}
58+
}
59+
60+
const filtered = uniq(redundancyVideos.map(r => r.strategy))
61+
if (filtered.length !== redundancyVideos.length) {
62+
return 'Redundancy video entries should have unique strategies'
63+
}
64+
65+
const recentlyAddedStrategy = redundancyVideos.find(r => r.strategy === 'recently-added') as RecentlyAddedStrategy
66+
if (recentlyAddedStrategy && isNaN(recentlyAddedStrategy.minViews)) {
67+
return 'Min views in recently added strategy is not a number'
68+
}
69+
}
70+
71+
if (isProdInstance()) {
72+
const configStorage = config.get('storage')
73+
for (const key of Object.keys(configStorage)) {
74+
if (configStorage[key].startsWith('storage/')) {
75+
logger.warn(
76+
'Directory of %s should not be in the production directory of PeerTube. Please check your production configuration file.',
77+
key
78+
)
79+
}
80+
}
81+
}
82+
83+
return null
84+
}
85+
86+
// We get db by param to not import it in this file (import orders)
87+
async function clientsExist () {
88+
const totalClients = await OAuthClientModel.countTotal()
89+
90+
return totalClients !== 0
91+
}
92+
93+
// We get db by param to not import it in this file (import orders)
94+
async function usersExist () {
95+
const totalUsers = await UserModel.countTotal()
96+
97+
return totalUsers !== 0
98+
}
99+
100+
// We get db by param to not import it in this file (import orders)
101+
async function applicationExist () {
102+
const totalApplication = await ApplicationModel.countTotal()
103+
104+
return totalApplication !== 0
105+
}
106+
107+
// ---------------------------------------------------------------------------
108+
109+
export {
110+
checkConfig,
111+
clientsExist,
112+
usersExist,
113+
applicationExist,
114+
checkActivityPubUrls
115+
}

‎server/initializers/checker.ts ‎server/initializers/checker-before-init.ts

+11-99
Original file line numberDiff line numberDiff line change
@@ -1,78 +1,8 @@
11
import * as config from 'config'
2-
import { promisify0, isProdInstance } from '../helpers/core-utils'
3-
import { UserModel } from '../models/account/user'
4-
import { ApplicationModel } from '../models/application/application'
5-
import { OAuthClientModel } from '../models/oauth/oauth-client'
6-
import { parse } from 'url'
7-
import { CONFIG } from './constants'
8-
import { logger } from '../helpers/logger'
9-
import { getServerActor } from '../helpers/utils'
10-
import { RecentlyAddedStrategy, VideosRedundancy } from '../../shared/models/redundancy'
2+
import { promisify0 } from '../helpers/core-utils'
113
import { isArray } from '../helpers/custom-validators/misc'
12-
import { uniq } from 'lodash'
13-
14-
async function checkActivityPubUrls () {
15-
const actor = await getServerActor()
16-
17-
const parsed = parse(actor.url)
18-
if (CONFIG.WEBSERVER.HOST !== parsed.host) {
19-
const NODE_ENV = config.util.getEnv('NODE_ENV')
20-
const NODE_CONFIG_DIR = config.util.getEnv('NODE_CONFIG_DIR')
21-
22-
logger.warn(
23-
'It seems PeerTube was started (and created some data) with another domain name. ' +
24-
'This means you will not be able to federate! ' +
25-
'Please use %s %s npm run update-host to fix this.',
26-
NODE_CONFIG_DIR ? `NODE_CONFIG_DIR=${NODE_CONFIG_DIR}` : '',
27-
NODE_ENV ? `NODE_ENV=${NODE_ENV}` : ''
28-
)
29-
}
30-
}
314

32-
// Some checks on configuration files
33-
// Return an error message, or null if everything is okay
34-
function checkConfig () {
35-
const defaultNSFWPolicy = config.get<string>('instance.default_nsfw_policy')
36-
37-
// NSFW policy
38-
if ([ 'do_not_list', 'blur', 'display' ].indexOf(defaultNSFWPolicy) === -1) {
39-
return 'NSFW policy setting should be "do_not_list" or "blur" or "display" instead of ' + defaultNSFWPolicy
40-
}
41-
42-
// Redundancies
43-
const redundancyVideos = config.get<VideosRedundancy[]>('redundancy.videos.strategies')
44-
if (isArray(redundancyVideos)) {
45-
for (const r of redundancyVideos) {
46-
if ([ 'most-views', 'trending', 'recently-added' ].indexOf(r.strategy) === -1) {
47-
return 'Redundancy video entries should have "most-views" strategy instead of ' + r.strategy
48-
}
49-
}
50-
51-
const filtered = uniq(redundancyVideos.map(r => r.strategy))
52-
if (filtered.length !== redundancyVideos.length) {
53-
return 'Redundancy video entries should have unique strategies'
54-
}
55-
56-
const recentlyAddedStrategy = redundancyVideos.find(r => r.strategy === 'recently-added') as RecentlyAddedStrategy
57-
if (recentlyAddedStrategy && isNaN(recentlyAddedStrategy.minViews)) {
58-
return 'Min views in recently added strategy is not a number'
59-
}
60-
}
61-
62-
if (isProdInstance()) {
63-
const configStorage = config.get('storage')
64-
for (const key of Object.keys(configStorage)) {
65-
if (configStorage[key].startsWith('storage/')) {
66-
logger.warn(
67-
'Directory of %s should not be in the production directory of PeerTube. Please check your production configuration file.',
68-
key
69-
)
70-
}
71-
}
72-
}
73-
74-
return null
75-
}
5+
// ONLY USE CORE MODULES IN THIS FILE!
766

777
// Check the config files
788
function checkMissedConfig () {
@@ -109,6 +39,14 @@ function checkMissedConfig () {
10939
}
11040
}
11141

42+
const redundancyVideos = config.get<any>('redundancy.videos.strategies')
43+
if (isArray(redundancyVideos)) {
44+
for (const r of redundancyVideos) {
45+
if (!r.size) miss.push('redundancy.videos.strategies.size')
46+
if (!r.min_lifetime) miss.push('redundancy.videos.strategies.min_lifetime')
47+
}
48+
}
49+
11250
const missingAlternatives = requiredAlternatives.filter(
11351
set => !set.find(alternative => !alternative.find(key => !config.has(key)))
11452
)
@@ -163,36 +101,10 @@ async function checkFFmpegEncoders (): Promise<Map<string, boolean>> {
163101
}
164102
}
165103

166-
// We get db by param to not import it in this file (import orders)
167-
async function clientsExist () {
168-
const totalClients = await OAuthClientModel.countTotal()
169-
170-
return totalClients !== 0
171-
}
172-
173-
// We get db by param to not import it in this file (import orders)
174-
async function usersExist () {
175-
const totalUsers = await UserModel.countTotal()
176-
177-
return totalUsers !== 0
178-
}
179-
180-
// We get db by param to not import it in this file (import orders)
181-
async function applicationExist () {
182-
const totalApplication = await ApplicationModel.countTotal()
183-
184-
return totalApplication !== 0
185-
}
186-
187104
// ---------------------------------------------------------------------------
188105

189106
export {
190-
checkConfig,
191107
checkFFmpeg,
192108
checkFFmpegEncoders,
193-
checkMissedConfig,
194-
clientsExist,
195-
usersExist,
196-
applicationExist,
197-
checkActivityPubUrls
109+
checkMissedConfig
198110
}

0 commit comments

Comments
 (0)