I would be interested in how to circumvent this for future analysis.
You can query by year, and then aggregate the years. From a past project, in nodejs:
/* Imports */ import fs from "fs" import axios from "axios" /* Utilities */ let print = console.log; let sleep = (ms) => new Promise(resolve => setTimeout(resolve, ms)) /* Support function */ let graphQLendpoint = 'https://www.forum.effectivealtruism.org/graphql/′ async function fetchEAForumPosts(start, end){ let response = await axios(graphQLendpoint, ({ method: ‘POST’, headers: ({ ‘Content-Type’: ‘application/json’ }), data: JSON.stringify(({ query: ` { posts(input: { terms: { after: “${start}” before: “${end}” } enableTotal: true }) { totalCount results{ pageUrl user { slug karma } } } }` })), })) .then(res ⇒ res ? res.data ? res.data.data ? res.data.data.posts ? res.data.data.posts.results : null : null : null : null) return response } /* Body */ let years = []; for (var i = 2005; i ⇐ 2021; i++) { years.push(i); } // Example, getting only 1 year. let main0 = async () ⇒ { let data = await fetchEAForumPosts(“2005-01-01“,”2006-01-01”) console.log(JSON.stringify(data,null,2)) } //main0() // Actual body let main = async () ⇒ { let results = [] for(let year of years){ print(year) let firstDayOfYear = `${year}-01-01` let firstDayOfNextYear = `${year+1}-01-01` let data = await fetchEAForumPosts(firstDayOfYear, firstDayOfNextYear) //console.log(JSON.stringify(data,null,2)) //console.log(data.slice(0,5)) results.push(...data) await sleep(5000) } print(results) fs.writeFileSync(“eaforumposts.json”, JSON.stringify(results, 0, 2)) } main()
Neat, thank you!
You can query by year, and then aggregate the years. From a past project, in nodejs:
Neat, thank you!