Skip to content

Commit 6a5f892

Browse files
author
touchRED
committed
fix: update tests to remove dayjs where possible, update formats
1 parent 98b8f69 commit 6a5f892

File tree

29 files changed

+43
-81
lines changed

29 files changed

+43
-81
lines changed

‎package.json

+1-3
Original file line numberDiff line numberDiff line change
@@ -132,16 +132,14 @@
132132
},
133133
"bundleDependencies": [
134134
"jquery",
135-
"moment-timezone",
136135
"browser-request"
137136
],
138137
"browser": {
139138
"main": "./dist/mercury.web.js",
140139
"cheerio": "./src/shims/cheerio-query",
141140
"jquery": "./node_modules/jquery/dist/jquery.min.js",
142141
"postman-request": "browser-request",
143-
"iconv-lite": "./src/shims/iconv-lite",
144-
"moment-timezone": "./node_modules/moment-timezone/builds/moment-timezone-with-data-2012-2022.min.js"
142+
"iconv-lite": "./src/shims/iconv-lite"
145143
},
146144
"husky": {
147145
"hooks": {

‎src/cleaners/date-published.js

+11-5
Original file line numberDiff line numberDiff line change
@@ -45,11 +45,17 @@ export function createDate(dateString, timezone, format) {
4545
}
4646

4747
if (timezone) {
48-
return format
49-
? dayjs.tz(dateString, format, timezone)
50-
: dayjs.tz(new Date(dateString), timezone);
48+
try {
49+
return format
50+
? dayjs.tz(dateString, format, timezone)
51+
: dayjs.tz(dayjs(dateString).format('YYYY-MM-DD HH:mm:ss'), timezone);
52+
} catch (error) {
53+
// return an intentionally invalid dayjs object,
54+
// in case the input needs to be cleaned first
55+
return dayjs('');
56+
}
5157
}
52-
return format ? dayjs(dateString, format) : dayjs(new Date(dateString));
58+
return format ? dayjs(dateString, format) : dayjs(dateString);
5359
}
5460

5561
// Take a date published string, and hopefully return a date out of
@@ -70,7 +76,7 @@ export default function cleanDatePublished(
7076

7177
if (!date.isValid()) {
7278
dateString = cleanDateString(dateString);
73-
date = createDate(dateString, timezone, format);
79+
date = createDate(dateString, timezone);
7480
}
7581

7682
return date.isValid() ? date.toISOString() : null;

‎src/extractors/custom/clinicaltrials.gov/index.test.js

+1-2
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,6 @@
11
import assert from 'assert';
22
import URL from 'url';
33
import cheerio from 'cheerio';
4-
import moment from 'moment-timezone';
54

65
import Mercury from 'mercury';
76
import getExtractor from 'extractors/get-extractor';
@@ -59,7 +58,7 @@ describe('ClinicaltrialsGovExtractor', () => {
5958

6059
// Update these values with the expected values from
6160
// the article.
62-
assert.equal(moment(date_published).format('YYYY-MM-DD'), '2018-11-21');
61+
assert.equal(date_published, '2018-11-21T05:00:00.000Z');
6362
});
6463

6564
it('returns the content', async () => {

‎src/extractors/custom/fortune.com/index.js

-2
Original file line numberDiff line numberDiff line change
@@ -11,8 +11,6 @@ export const FortuneComExtractor = {
1111

1212
date_published: {
1313
selectors: ['.MblGHNMJ'],
14-
15-
timezone: 'UTC',
1614
},
1715

1816
lead_image_url: {

‎src/extractors/custom/genius.com/index.test.js

+1-3
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,6 @@
11
import assert from 'assert';
22
import URL from 'url';
33
import cheerio from 'cheerio';
4-
import moment from 'moment';
54

65
import Mercury from 'mercury';
76
import getExtractor from 'extractors/get-extractor';
@@ -51,11 +50,10 @@ describe('GeniusComExtractor', () => {
5150
// To pass this test, fill out the date_published selector
5251
// in ./src/extractors/custom/genius.com/index.js.
5352
const { date_published } = await result;
54-
const newDatePublished = moment(date_published).format();
5553

5654
// Update these values with the expected values from
5755
// the article.
58-
assert.equal(newDatePublished.split('T')[0], '1984-06-25');
56+
assert.equal(date_published, '1984-06-25T04:00:00.000Z');
5957
});
6058

6159
it('returns the lead_image_url', async () => {

‎src/extractors/custom/news.nationalgeographic.com/index.js

+1-2
Original file line numberDiff line numberDiff line change
@@ -11,8 +11,7 @@ export const NewsNationalgeographicComExtractor = {
1111

1212
date_published: {
1313
selectors: [['meta[name="article:published_time"]', 'value']],
14-
format: 'ddd MMM DD HH:mm:ss zz YYYY',
15-
timezone: 'EST',
14+
timezone: 'America/New_York',
1615
},
1716

1817
dek: {

‎src/extractors/custom/news.nationalgeographic.com/index.test.js

+1-1
Original file line numberDiff line numberDiff line change
@@ -49,7 +49,7 @@ describe('NewsNationalgeographicComExtractor', () => {
4949

5050
// Update these values with the expected values from
5151
// the article.
52-
assert.equal(date_published, '2015-08-03T17:45:00.000Z');
52+
assert.equal(date_published, '2015-08-03T16:45:00.000Z');
5353
});
5454

5555
it('returns the dek', async () => {

‎src/extractors/custom/people.com/index.test.js

+1-6
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,6 @@
11
import assert from 'assert';
22
import URL from 'url';
33
import cheerio from 'cheerio';
4-
import moment from 'moment-timezone';
54

65
import Mercury from 'mercury';
76
import getExtractor from 'extractors/get-extractor';
@@ -55,13 +54,9 @@ describe('PeopleComExtractor', () => {
5554
// in ./src/extractors/custom/people.com/index.js.
5655
const { date_published } = await result;
5756

58-
const new_date_published = moment(date_published)
59-
.format()
60-
.split('T')[0];
61-
6257
// Update these values with the expected values from
6358
// the article.
64-
assert.equal(new_date_published, '2016-12-12');
59+
assert.equal(date_published, '2016-12-12T14:22:00.000Z');
6560
});
6661

6762
it('returns the lead_image_url', async () => {

‎src/extractors/custom/pitchfork.com/index.test.js

+1-5
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,6 @@
11
import assert from 'assert';
22
import URL from 'url';
33
import cheerio from 'cheerio';
4-
import moment from 'moment-timezone';
54

65
import Mercury from 'mercury';
76
import getExtractor from 'extractors/get-extractor';
@@ -41,11 +40,8 @@ describe('PitchforkComExtractor', () => {
4140

4241
it('returns the date_published', async () => {
4342
const { date_published } = await result;
44-
const new_date_published = moment(date_published)
45-
.format()
46-
.split('T')[0];
4743

48-
assert.equal(new_date_published, '2019-06-07');
44+
assert.equal(date_published, '2019-06-07T04:00:00.000Z');
4945
});
5046

5147
it('returns the dek', async () => {

‎src/extractors/custom/takagi-hiromitsu.jp/index.test.js

+1-3
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,6 @@
11
import assert from 'assert';
22
import URL from 'url';
33
import cheerio from 'cheerio';
4-
import moment from 'moment';
54

65
import Mercury from 'mercury';
76
import getExtractor from 'extractors/get-extractor';
@@ -57,11 +56,10 @@ describe('TakagihiromitsuJpExtractor', () => {
5756
// To pass this test, fill out the date_published selector
5857
// in ./src/extractors/custom/takagi-hiromitsu.jp/index.js.
5958
const { date_published } = await result;
60-
const newDatePublished = moment(date_published).format();
6159

6260
// Update these values with the expected values from
6361
// the article.
64-
assert.equal(newDatePublished.split('T')[0], '2019-02-17');
62+
assert.equal(date_published, '2019-02-17T14:34:06.000Z');
6563
});
6664

6765
it('returns the dek', async () => {

‎src/extractors/custom/www.chicagotribune.com/index.js

+1
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ export const WwwChicagotribuneComExtractor = {
1111

1212
date_published: {
1313
selectors: ['time'],
14+
timezone: 'America/Chicago',
1415
},
1516

1617
lead_image_url: {

‎src/extractors/custom/www.chicagotribune.com/index.test.js

+1-5
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,6 @@
11
import assert from 'assert';
22
import URL from 'url';
33
import cheerio from 'cheerio';
4-
import moment from 'moment-timezone';
54

65
import Mercury from 'mercury';
76
import getExtractor from 'extractors/get-extractor';
@@ -55,13 +54,10 @@ describe('WwwChicagotribuneComExtractor', () => {
5554
// To pass this test, fill out the date_published selector
5655
// in ./src/extractors/custom/www.chicagotribune.com/index.js.
5756
const { date_published } = await result;
58-
const new_date_published = moment(date_published)
59-
.format()
60-
.split('T')[0];
6157

6258
// Update these values with the expected values from
6359
// the article.
64-
assert.equal(new_date_published, '2016-12-13');
60+
assert.equal(date_published, '2016-12-13T21:45:00.000Z');
6561
});
6662

6763
it('returns the lead_image_url', async () => {

‎src/extractors/custom/www.infoq.com/index.js

+1-1
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@ export const WwwInfoqComExtractor = {
1111

1212
date_published: {
1313
selectors: ['.article__readTime.date'],
14-
format: 'YYYY年MM月DD日',
14+
format: 'YYYY[年]M[月]D[日]',
1515
timezone: 'Asia/Tokyo',
1616
},
1717

‎src/extractors/custom/www.macrumors.com/index.js

+1-2
Original file line numberDiff line numberDiff line change
@@ -11,8 +11,7 @@ export const WwwMacrumorsComExtractor = {
1111

1212
date_published: {
1313
selectors: [['time', 'datetime']],
14-
15-
timezone: 'America/Los_Angeles',
14+
// timezone: 'America/Los_Angeles',
1615
},
1716

1817
dek: {

‎src/extractors/custom/www.msn.com/index.test.js

+1-3
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,6 @@
11
import assert from 'assert';
22
import URL from 'url';
33
import cheerio from 'cheerio';
4-
import moment from 'moment';
54

65
import Mercury from 'mercury';
76
import getExtractor from 'extractors/get-extractor';
@@ -58,11 +57,10 @@ describe('MSNExtractor', () => {
5857
// To pass this test, fill out the date_published selector
5958
// in ./src/extractors/custom/www.msn.com/index.js.
6059
const { date_published } = await result;
61-
const newDatePublished = moment(date_published).format();
6260

6361
// Update these values with the expected values from
6462
// the article.
65-
assert.equal(newDatePublished.split('T')[0], '2016-09-21');
63+
assert.equal(date_published.split('T')[0], '2016-09-21');
6664
});
6765

6866
it('returns the lead_image_url', async () => {

‎src/extractors/custom/www.nationalgeographic.com/index.test.js

+1-5
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,6 @@
11
import assert from 'assert';
22
import URL from 'url';
33
import cheerio from 'cheerio';
4-
import moment from 'moment-timezone';
54

65
import Mercury from 'mercury';
76
import getExtractor from 'extractors/get-extractor';
@@ -44,13 +43,10 @@ describe('WwwNationalgeographicComExtractor', () => {
4443
// To pass this test, fill out the date_published selector
4544
// in ./src/extractors/custom/www.nationalgeographic.com/index.js.
4645
const { date_published } = await result;
47-
const new_date_published = moment(date_published)
48-
.format()
49-
.split('T')[0];
5046

5147
// Update these values with the expected values from
5248
// the article.
53-
assert.equal(new_date_published, '2016-12-15');
49+
assert.equal(date_published.split('T')[0], '2016-12-15');
5450
});
5551

5652
it('returns the dek', async () => {

‎src/extractors/custom/www.nbcnews.com/index.js

+1-1
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@ export const WwwNbcnewsComExtractor = {
1919
'.flag_article-wrapper time',
2020
],
2121

22-
timezone: 'America/New_York',
22+
// timezone: 'America/New_York',
2323
},
2424

2525
lead_image_url: {

‎src/extractors/custom/www.nbcnews.com/index.test.js

+1-1
Original file line numberDiff line numberDiff line change
@@ -53,7 +53,7 @@ describe('WwwNbcnewsComExtractor', () => {
5353

5454
// Update these values with the expected values from
5555
// the article.
56-
assert.equal(date_published, '2016-12-13T23:06:00.000Z');
56+
assert.equal(date_published, '2016-12-13T18:06:00.000Z');
5757
});
5858

5959
it('returns the lead_image_url', async () => {

‎src/extractors/custom/www.nydailynews.com/index.test.js

+1-5
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,6 @@
11
import assert from 'assert';
22
import URL from 'url';
33
import cheerio from 'cheerio';
4-
import moment from 'moment-timezone';
54

65
import Mercury from 'mercury';
76
import getExtractor from 'extractors/get-extractor';
@@ -55,14 +54,11 @@ describe('WwwNydailynewsComExtractor', () => {
5554
// To pass this test, fill out the date_published selector
5655
// in ./src/extractors/custom/www.nydailynews.com/index.js.
5756
const { date_published } = await result;
58-
const new_date_published = moment(date_published)
59-
.format()
60-
.split('T')[0];
6157

6258
// Update these values with the expected values from
6359
// the article.
6460

65-
assert.equal(new_date_published, '2016-12-16');
61+
assert.equal(date_published.split('T')[0], '2016-12-16');
6662
});
6763

6864
it('returns the lead_image_url', async () => {

‎src/extractors/custom/www.ossnews.jp/index.js

+1-1
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@ export const WwwOssnewsJpExtractor = {
99

1010
date_published: {
1111
selectors: ['p.fs12'],
12-
format: 'YYYY年MM月DD日 HH:mm',
12+
format: 'YYYY[年]M[月]D[日] HH:mm',
1313
timezone: 'Asia/Tokyo',
1414
},
1515

‎src/extractors/custom/www.phoronix.com/index.js

+1-1
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ export const WwwPhoronixComExtractor = {
1212
date_published: {
1313
selectors: ['.author'],
1414
// 1 June 2019 at 08:34 PM EDT
15-
format: 'D MMMM YYYY at hh:mm',
15+
format: 'D MMMM YYYY [at] hh:mm A',
1616
timezone: 'America/New_York',
1717
},
1818

‎src/extractors/custom/www.phoronix.com/index.test.js

+1-1
Original file line numberDiff line numberDiff line change
@@ -57,7 +57,7 @@ describe('WwwPhoronixComExtractor', () => {
5757

5858
// Update these values with the expected values from
5959
// the article.
60-
assert.equal(date_published, '2019-06-01T12:34:00.000Z');
60+
assert.equal(date_published, '2019-06-02T00:34:00.000Z');
6161
});
6262

6363
it('returns the dek', async () => {

‎src/extractors/custom/www.politico.com/index.test.js

+1-5
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,6 @@
11
import assert from 'assert';
22
import URL from 'url';
33
import cheerio from 'cheerio';
4-
import moment from 'moment-timezone';
54

65
import Mercury from 'mercury';
76
import getExtractor from 'extractors/get-extractor';
@@ -55,13 +54,10 @@ describe('PoliticoExtractor', () => {
5554
// To pass this test, fill out the date_published selector
5655
// in ./src/extractors/custom/www.politico.com/index.js.
5756
const { date_published } = await result;
58-
const new_date_published = moment(date_published)
59-
.format()
60-
.split('T')[0];
6157

6258
// Update these values with the expected values from
6359
// the article.
64-
assert.equal(new_date_published, '2016-10-04');
60+
assert.equal(date_published.split('T')[0], '2016-10-04');
6561
});
6662

6763
it('returns the lead_image_url', async () => {

‎src/extractors/custom/www.prospectmagazine.co.uk/index.js

-2
Original file line numberDiff line numberDiff line change
@@ -11,8 +11,6 @@ export const WwwProspectmagazineCoUkExtractor = {
1111

1212
date_published: {
1313
selectors: [['meta[name="article:published_time"]', 'value'], '.post-info'],
14-
15-
timezone: 'Europe/London',
1614
},
1715

1816
dek: {

0 commit comments

Comments
 (0)