NunoSempere comments on Getting a feel for changes of karma and controversy in the EA Forum over time

[NunoSempere]

I would be interested in how to circumvent this for future analysis.

You can query by year, and then aggregate the years. From a past project, in nodejs:

/* Imports */
import fs from "fs"
import axios from "axios"

/* Utilities */
let print = console.log;
let sleep = (ms) => new Promise(resolve => setTimeout(resolve, ms))

/* Support function */
let graphQLendpoint = 'https://​​www.forum.effectivealtruism.org/​​graphql/​​′
async function fetchEAForumPosts(start, end){
  let response  = await axios(graphQLendpoint, ({
    method: ‘POST’,
    headers: ({ ‘Content-Type’: ‘application/​json’ }),
    data: JSON.stringify(({ query: `
       {
        posts(input: {
          terms: {
          after: “${start}”
          before: “${end}”
          }
          enableTotal: true
        }) {
          totalCount
          results{
            pageUrl
            user {
              slug
              karma
            }
            
          }
        }
      }`
})),
  }))
  .then(res ⇒ res ? res.data ? res.data.data ? res.data.data.posts ? res.data.data.posts.results : null : null : null : null)
  return response
}

/​* Body */​
let years = [];
for (var i = 2005; i ⇐ 2021; i++) {
   years.push(i);
}

//​ Example, getting only 1 year.
let main0 = async () ⇒ {
  let data = await fetchEAForumPosts(“2005-01-01“,”2006-01-01”)
  console.log(JSON.stringify(data,null,2))
}
//​main0()

//​ Actual body
let main = async () ⇒ {
  let results = []
  for(let year of years){
    print(year)
    let firstDayOfYear = `${year}-01-01`
    let firstDayOfNextYear = `${year+1}-01-01`
    let data = await fetchEAForumPosts(firstDayOfYear, firstDayOfNextYear)
    //​console.log(JSON.stringify(data,null,2))
    //​console.log(data.slice(0,5))
    results.push(...data)
    await sleep(5000)
  }
  print(results)
  fs.writeFileSync(“eaforumposts.json”, JSON.stringify(results, 0, 2))
}
main()

[FJehn]

Neat, thank you!