MathJax.Hub.Config({ extensions: ['tex2jax.js'], jax: ['input/TeX','output/HTML-CSS'], tex2jax: { inlineMath: [ ['$$$','$$$'] ], processEscapes: true, processClass: 'tex2jax', ignoreClass: 'html' }, showProcessingMessages: false, messageStyle: 'none' });           External Reviews of GiveWell's Research | GiveWell          @import url("https://web.archive.org./web/20190428104243cs_/https://www.givewell.org/modules/system/system.base.css?pqo29p"); @import url("https://web.archive.org./web/20190428104243cs_/https://www.givewell.org/modules/system/system.menus.css?pqo29p"); @import url("https://web.archive.org./web/20190428104243cs_/https://www.givewell.org/modules/system/system.messages.css?pqo29p"); @import url("https://web.archive.org./web/20190428104243cs_/https://www.givewell.org/modules/system/system.theme.css?pqo29p");   @import url("https://web.archive.org./web/20190428104243cs_/https://www.givewell.org/sites/all/modules/contrib/jquery_update/replace/ui/themes/base/minified/jquery.ui.core.min.css?pqo29p"); @import url("https://web.archive.org./web/20190428104243cs_/https://www.givewell.org/sites/all/modules/contrib/jquery_update/replace/ui/themes/base/minified/jquery.ui.theme.min.css?pqo29p"); @import url("https://web.archive.org./web/20190428104243cs_/https://www.givewell.org/sites/all/modules/contrib/jquery_update/replace/ui/themes/base/minified/jquery.ui.accordion.min.css?pqo29p");   @import url("https://web.archive.org./web/20190428104243cs_/https://www.givewell.org/modules/aggregator/aggregator.css?pqo29p"); @import url("https://web.archive.org./web/20190428104243cs_/https://www.givewell.org/modules/comment/comment.css?pqo29p"); @import url("https://web.archive.org./web/20190428104243cs_/https://www.givewell.org/sites/all/modules/contrib/date/date_api/date.css?pqo29p"); @import url("https://web.archive.org./web/20190428104243cs_/https://www.givewell.org/modules/field/theme/field.css?pqo29p"); @import url("https://web.archive.org./web/20190428104243cs_/https://www.givewell.org/sites/all/modules/contrib/footnotes/footnotes.css?pqo29p"); @import url("https://web.archive.org./web/20190428104243cs_/https://www.givewell.org/modules/node/node.css?pqo29p"); @import url("https://web.archive.org./web/20190428104243cs_/https://www.givewell.org/modules/search/search.css?pqo29p"); @import url("https://web.archive.org./web/20190428104243cs_/https://www.givewell.org/modules/user/user.css?pqo29p"); @import url("https://web.archive.org./web/20190428104243cs_/https://www.givewell.org/sites/all/modules/contrib/views/css/views.css?pqo29p"); @import url("https://web.archive.org./web/20190428104243cs_/https://www.givewell.org/sites/all/modules/contrib/ckeditor/css/ckeditor.css?pqo29p");   @import url("https://web.archive.org./web/20190428104243cs_/https://www.givewell.org/sites/all/modules/contrib/ctools/css/ctools.css?pqo29p"); @import url("https://web.archive.org./web/20190428104243cs_/https://www.givewell.org/sites/all/modules/contrib/taxonomy_access/taxonomy_access.css?pqo29p"); @import url("https://web.archive.org./web/20190428104243cs_/https://www.givewell.org/sites/all/modules/contrib/content_type_extras/css/content_type_extras.css?pqo29p"); @import url("https://web.archive.org./web/20190428104243cs_/https://www.givewell.org/sites/all/modules/contrib/tableofcontents/tableofcontents.css?pqo29p"); @import url("https://web.archive.org./web/20190428104243cs_/https://www.givewell.org/sites/all/modules/contrib/print/print_ui/css/print_ui.theme.css?pqo29p");   @import url("https://web.archive.org./web/20190428104243cs_/https://www.givewell.org/sites/all/themes/gw_basic/styles/css/global/normalize.css?pqo29p"); @import url("https://web.archive.org./web/20190428104243cs_/https://www.givewell.org/sites/all/themes/gw_basic/styles/css/global/gw-fonts.css?pqo29p"); @import url("https://web.archive.org./web/20190428104243cs_/https://www.givewell.org/sites/all/themes/gw_basic/styles/css/global/html.css?pqo29p"); @import url("https://web.archive.org./web/20190428104243cs_/https://www.givewell.org/sites/all/themes/gw_basic/styles/css/global/global.css?pqo29p"); @import url("https://web.archive.org./web/20190428104243cs_/https://www.givewell.org/sites/all/themes/gw_basic/styles/css/global/meanmenu.css?pqo29p"); @import url("https://web.archive.org./web/20190428104243cs_/https://www.givewell.org/sites/all/themes/gw_basic/styles/css/global/page-layout.css?pqo29p"); @import url("https://web.archive.org./web/20190428104243cs_/https://www.givewell.org/sites/all/themes/gw_basic/styles/css/global/html-colors.css?pqo29p"); @import url("https://web.archive.org./web/20190428104243cs_/https://www.givewell.org/sites/all/themes/gw_basic/styles/css/pages/pages.css?pqo29p"); @import url("https://web.archive.org./web/20190428104243cs_/https://www.givewell.org/sites/all/themes/gw_basic/styles/sitesearch.css?pqo29p"); @import url("https://web.archive.org./web/20190428104243cs_/https://www.givewell.org/sites/all/themes/gw_basic/styles/formstack.css?pqo29p");   @import url("https://web.archive.org./web/20190428104243cs_/https://www.givewell.org/sites/all/themes/gw_basic/styles/css/global/admin.css?pqo29p");  dataLayer = [{"transactionId":0,"transactionAffiliation":"unknown","transactionTotal":0,"transactionProducts":[]}]                              jQuery.extend(Drupal.settings, {"basePath":"\/","pathPrefix":"","ajaxPageState":{"theme":"gw_basic","theme_token":"UMcAIajX46KQIZA5bK5vN1kYO5EDED9Q4SjZcTH3VOw","js":{"0":1,"public:\/\/google_tag\/google_tag.script.js":1,"sites\/all\/modules\/contrib\/jquery_update\/replace\/jquery\/1.10\/jquery.min.js":1,"misc\/jquery-extend-3.4.0.js":1,"misc\/jquery.once.js":1,"misc\/drupal.js":1,"sites\/all\/modules\/contrib\/jquery_update\/replace\/ui\/ui\/minified\/jquery.ui.core.min.js":1,"sites\/all\/modules\/contrib\/jquery_update\/replace\/ui\/ui\/minified\/jquery.ui.widget.min.js":1,"sites\/all\/modules\/contrib\/jquery_update\/replace\/ui\/external\/jquery.cookie.js":1,"sites\/all\/modules\/contrib\/jquery_update\/replace\/ui\/ui\/minified\/jquery.ui.accordion.min.js":1,"sites\/all\/modules\/contrib\/mathjax\/mathjax.js":1,"sites\/all\/modules\/contrib\/resp_img\/resp_img.js":1,"sites\/all\/modules\/contrib\/tableofcontents\/js\/jquery.scrollTo-min.js":1,"sites\/all\/modules\/contrib\/tableofcontents\/js\/jquery.localscroll-min.js":1,"sites\/all\/modules\/contrib\/tableofcontents\/js\/tableofcontents.js":1,"sites\/all\/modules\/custom\/givewell_add_querystring\/js\/givewell_add_querystring.js":1,"https:\/\/web.archive.org.\/web\/20190428104243\/https:\/\/cdnjs.cloudflare.com\/ajax\/libs\/mathjax\/2.7.0\/MathJax.js?config=TeX-AMS-MML_HTMLorMML":1,"sites\/all\/themes\/gw_basic\/scripts\/lib\/modernizr.custom.74085.js":1,"sites\/all\/themes\/gw_basic\/scripts\/lib\/modernizr.flexbox.js":1,"sites\/all\/themes\/gw_basic\/scripts\/lib\/mean-menu\/jquery.meanmenu.min.js":1,"sites\/all\/themes\/gw_basic\/scripts\/lib\/carousel\/responsive-carousel.js":1,"sites\/all\/themes\/gw_basic\/scripts\/lib\/carousel\/responsive-carousel.pagination.js":1,"sites\/all\/themes\/gw_basic\/scripts\/lib\/carousel\/responsive-carousel.autoplay.js":1,"sites\/all\/themes\/gw_basic\/scripts\/lib\/carousel\/responsive-carousel.loop.js":1,"sites\/all\/themes\/gw_basic\/scripts\/lib\/jquery-migrate-1.0.0.js":1,"sites\/all\/themes\/gw_basic\/scripts\/lib\/jquery-validation\/dist\/jquery.validate.min.js":1,"sites\/all\/themes\/gw_basic\/scripts\/lib\/multiselect\/jquery.multiselect.min.js":1,"sites\/all\/themes\/gw_basic\/scripts\/lib\/dropkick.js":1,"sites\/all\/themes\/gw_basic\/scripts\/lib\/jquery.pep.js":1,"sites\/all\/themes\/gw_basic\/scripts\/main.js":1},"css":{"modules\/system\/system.base.css":1,"modules\/system\/system.menus.css":1,"modules\/system\/system.messages.css":1,"modules\/system\/system.theme.css":1,"misc\/ui\/jquery.ui.core.css":1,"misc\/ui\/jquery.ui.theme.css":1,"misc\/ui\/jquery.ui.accordion.css":1,"modules\/aggregator\/aggregator.css":1,"modules\/comment\/comment.css":1,"sites\/all\/modules\/contrib\/date\/date_api\/date.css":1,"modules\/field\/theme\/field.css":1,"sites\/all\/modules\/contrib\/footnotes\/footnotes.css":1,"modules\/node\/node.css":1,"modules\/search\/search.css":1,"modules\/user\/user.css":1,"sites\/all\/modules\/contrib\/views\/css\/views.css":1,"sites\/all\/modules\/contrib\/ckeditor\/css\/ckeditor.css":1,"sites\/all\/modules\/contrib\/ctools\/css\/ctools.css":1,"sites\/all\/modules\/contrib\/taxonomy_access\/taxonomy_access.css":1,"sites\/all\/modules\/contrib\/content_type_extras\/css\/content_type_extras.css":1,"sites\/all\/modules\/contrib\/tableofcontents\/tableofcontents.css":1,"sites\/all\/modules\/contrib\/print\/print_ui\/css\/print_ui.theme.css":1,"sites\/all\/themes\/gw_basic\/styles\/css\/global\/normalize.css":1,"sites\/all\/themes\/gw_basic\/styles\/css\/global\/gw-fonts.css":1,"sites\/all\/themes\/gw_basic\/styles\/css\/global\/html.css":1,"sites\/all\/themes\/gw_basic\/styles\/css\/global\/global.css":1,"sites\/all\/themes\/gw_basic\/styles\/css\/global\/meanmenu.css":1,"sites\/all\/themes\/gw_basic\/styles\/css\/global\/page-layout.css":1,"sites\/all\/themes\/gw_basic\/styles\/css\/global\/html-colors.css":1,"sites\/all\/themes\/gw_basic\/styles\/css\/pages\/pages.css":1,"sites\/all\/themes\/gw_basic\/styles\/sitesearch.css":1,"sites\/all\/themes\/gw_basic\/styles\/formstack.css":1,"sites\/all\/themes\/gw_basic\/styles\/css\/global\/admin.css":1}},"respImg":{"default_suffix":"","current_suffix":false,"forceRedirect":false,"forceResize":false,"reloadOnResize":false,"useDevicePixelRatio":false,"suffixes":{"":1}},"tableofcontents":{"collapse":false,"scroll":true},"urlIsAjaxTrusted":{"\/about\/self-evaluation\/external-reviews":true}});     Jump to Navigation   Google Tag Manager          GiveWell        Giving EffectivelyGiving 101 Your Donation Can Change Someone's Life The Wrong Donation Can Accomplish Nothing Your Dollar Goes Further Overseas Your Dollar Goes Further When You Fund the Right Program Quick Start Guide  How We WorkWho We Are Process Criteria Transparency Evaluations of GiveWell Research FAQ Core Competencies  Top CharitiesMalaria Consortium Deworm the World Initiative Helen Keller International Against Malaria Foundation Schistosomiasis Control Initiative Sightsavers The END Fund GiveDirectly Standout Charities  ResearchCharity Site Visits Notes from Research Conversations Intervention Reports Cost-Effectiveness Analyses Other Charity Reviews GiveWell Incubation Grants  Our Mistakes AboutAbout GiveWell Progress to Date Our Story Our People Official Records Reputation Impact Frequently Asked Questions Donate Contact Us Jobs  UpdatesBlog RSS Feeds Stay Updated           Enter search terms here.   This search returns results from both GiveWell's main site and from the GiveWell Blog. 

  Search form  Search             DonateDonate

             You are hereHome » How We Work » Evaluations of GiveWell » External Reviews of GiveWell's Research External Reviews of GiveWell's Research     FacebookTwitter>Print>Email                   We encourage feedback on our research in general, whether via our public research email list, the comments section of our blog, or direct contact. We have also pursued more in-depth, formal feedback from people who are unaffiliated with GiveWell. This page lays out the "assignments" that we have sent in order to structure this feedback, and also summarizes and links to completed submissions.

  Table of Contents   Issue overview assignment  Assignment description Completed assignments  Matthew Bonds, September 20 2010 - review of HIV/AIDS issue content Matthew Collin, October 19 2010 - review of Developing-world Education issue content Andy Beckingham, February 24, 2011 - review of maternal mortality issue content Pierre Thompson, December 23, 2011 - review of combination deworming issue content     Criteria & Heuristics assignment  Assignment description Completed assignments  Laura Freschi, August 12 2010 David Jinkins, July 14, 2011     Narrowing the Field assignment  Assignment description Completed assignments  Jonah Sinick, September 23 2010 Ju Min Kim, February 2, 2011     Charity review assignment  Assignment description Completed assignments  Robert Mundy, October 29, 2010: VillageReach Robert Mundy, December 4, 2010: Against Malaria Foundation Robert Mundy, December 15, 2010: Small Enterprise Foundation Jonah Sinick, January 1, 2011: Stop TB Partnership Wendy Knight, January 25, 2011: Nurse-Family Partnership Joshua Rosenberg, February 28, 2011: Village Enterprise Fund Kate Lang, April 29, 2011: Schistosomiasis Control Initiative Ju Min Kim, May 11, 2011: Chamroeun Rachel Gross, May 20, 2011: KIPP Lim Kwangjin, June 13, 2011: Stop TB Rachel Gross, September 5, 2011: Invest in Kids Ju Min Kim, November 12, 2011: VillageReach     Finding the Best Charity assignment  Assignment description Completed assignments  Stephanie Wykstra, July 21 2010 Robert Mundy, October 11, 2010 Timothy Moreland, December 17, 2010 Tobias Pfutze, March 2, 2011 Rachael Stephens, July 21, 2011     Unsolicited feedback  Gilles Corcos, November 1, 2012 FIECON, January 2018      Issue overview assignment Assignment description Full official assignment (DOC)

 The purpose of this assignment is to assess the conclusions we've come to (and the sources we've used) regarding key issues (from an individual donor standpoint) in international aid.

 For each iteration of this assignment, we collect all of the material relevant to a given issue into a single document. We request that the reviewer read this document and write their thoughts on the following questions: 

Are GiveWell's statements supported by credible and logical sources? Are there statements where other sources (more relevant/credible than what GiveWell used) contradict them? Is GiveWell's take generally consistent with a fair reading of the literature and available evidence? Other comments?Completed assignments  Matthew Bonds, September 20 2010 - review of HIV/AIDS issue content About Matthew Bonds

 HIV/AIDS content

 Full submission (DOC)

 In general, I thought the review of the cost-effectiveness of HIV/AIDS treatment and prevention strategies was excellent; clear, concise, and well-rooted in sensible literature sources. The general conclusion that many other forms of interventions may be under-funded relative to HIV/AIDS is a reasonable conclusion from this literature, and one that I happen to agree with.

 But I also think this is over-simplified to the point of losing sight of one of the important goals of treatment and prevention of HIV/AIDs: staving off negative indirect effects. There are two very important indirect effects of HIV: 1) secondary infections, and 2) economic impacts of losing productive laborers. If the HIV rates are on the rise, then the average infected individual infects more than one other individual, who then infect other individuals, and so on (if it’s declining than the average secondary infection is less than one, but still important). This effect is of course greater for high-risk adults than children, and higher for prevention than treatment (though treatment reduces transmission significantly as well). The epidemiological effects are then compounded by economic effects. Preventing a generation of orphans is even more important than saving the life of a single mother for her own sake. These issues are hard to measure, but they are not just abstract. And so, at the least, it would seem important to mention the importance of these other effects.

 There are only a few technical issues that I think could use some additional consideration, and I think there are also important questions about the inference one can draw about the conclusions of this issue review and how an individual donor can make an informed donations.

  Matthew Collin, October 19 2010 - review of Developing-world Education issue content About Matthew Collin

 Developing-world Education content

 Full submission (DOC)

 All together, I think the in-depth review of "developing-world education" covers all the main points I would have liked to see. On you two main questions: are you statements supported by credible evidence and are you interpreting it correctly? I think the answer to both of these questions is, for the most part, yes. Below I’ll throw in a few more references to check out. Also, I’ll suggest some more areas for you to pursue – even if you choose not to expand, the evidence you do cover is convincing enough. I’ll go into more detail below. 

  Andy Beckingham, February 24, 2011 - review of maternal mortality issue content Andy Beckingham FFPH is a Consultant in Public Health who has worked in the UK and India. He has over 23 years of experience in health care and is currently (as of January 2011) working with a range of agencies in Andhra Pradesh on joint efforts to improve the quality of maternal care in the state.

 Maternal mortality content

 Full submission (DOC)

 I found this to be a credible précis and explanation of the available evidence, written to assist potential donors. While not itself a systematic review nor using an explicit formal critical appraisal methodology, it is a very interesting pragmatic review. Its conclusion - that there is currently a shortage of good quality evidence from well-conducted trials that would demonstrably reduce maternal mortality in developing countries with poor health infrastructure - is a reasonable one. 

 I found the review to be structured in a logical way, and easy to read. I think that it provides potential donors with a very helpful summary of the available evidence, draws on credible sources of research to do so, interprets the evidence in the light of quality of evidence, and reduces the need for them to do their own complete search. 

 Pierre Thompson, December 23, 2011 - review of combination deworming issue content Pierre Thompson is a 2011 graduate of Georgetown University. He completed this assignment as part of his volunteer work for GiveWell.

 Combination deworming report

 Full submission (PDF)

 I believe that GiveWell has reached a reasonable assessment with this review, but the reasoning process itself could have been more lucid. At the end of the day, this review must make the case that deworming does (or does not) change lives for the better, and it should provide abundant evidence and logic to support that viewpoint. With respect to measuring impact, the evidence focuses largely on demonstrating improved health outcomes through randomized controlled trials; as such, the current evidence base does not balance internal and external validity. Considering a broader range of methodological and disciplinary perspectives, as well as alternative outcomes, could make GiveWell’s conclusions more robust. Donors also need practical information about how the intervention is designed and implemented, including challenges in the field – these details are too important to be implied. Finally, I maintain reasonable confidence in the cost- effectiveness analysis which GiveWell has provided, and do not expect the marginal adjustments I describe to change that assessment.

 No matter how strong, the evidence is only as good as the logic that synthesizes it. Much like a mathematical proof, the review should follow a clear line of reasoning and cite relevant evidence at every step. The statement which best describes the rationale for deworming is not found in the review itself, but in a later blog post about it. With great clarity, Elie writes that “[GiveWell’s] positive view of deworming stems from the fact that ... the intervention is so cheap that it is likely to be a relatively good buy and may be a great buy.” Thus, the burden of proof for supporting this particular intervention seems to depend on the “balance of probabilities” rather than “beyond reasonable doubt.” GiveWell would do well to emphasize that distinction. I feel that the “balance of probabilities” approach, which might entail looking at successful delivery of the intervention, has been shorted; this explains why important concepts, such as coverage rate and cure rate, are unfortunately relegated to the footnotes. Finally, there is a strong philosophical case to be made for this intervention because it is extremely cost-effective and the burden of disease is spread across many disenfranchised poor people. GiveWell should dedicate time and space to explain the principles of distributive justice which may have influenced its recommendation of this intervention.

 Criteria & Heuristics assignment Assignment description Full official assignment (DOC)

 The purpose of this assignment is to evaluate and reflect on (a) the criteria we use to evaluate charities; (b) the heuristics we use to identify charities most likely to meet these criteria.

 We first ask that the reviewer review the goals of our research and our high-level criteria and write thoughts on the following questions: 

Are GiveWell's criteria appropriate for individual donors (as defined above)? Are GiveWell's criteria appropriately defined and defended on the pages linked to above? Does the content of these pages make any claims that are unsupported (when support is claimed) or unreasonable?We then ask that the reviewer review our discussion of how we identified "priority programs", our list of "priority programs," and our description and defense of our heuristics for narrowing the field, and write thoughts on the following: 

For the purposes for which GiveWell is using "priority programs," has a reasonable process been used to identify "priority programs?" Should other programs be considered "priority" that are not? Should any of GiveWell's "priority programs" not be considered "priority?" Are the heuristics used by GiveWell reasonable for finding top contenders to meet the criteria above? Are other heuristics possible that might identify other strong charities?Completed assignments  Laura Freschi, August 12 2010 About Laura Freschi

 Full submission (DOC)

 Are GiveWell's criteria appropriate for individual donors (as defined above)? 

 Yes, I think they are.

 Are GiveWell's criteria appropriately defined and defended on the pages linked to above? 

 Yes, I think they are. However, I wonder if the documents rely a bit too much on links to GiveWell blog posts. I didn’t go on to read those links, partly because I wasn’t sure that was part of the “assignment” but also because I had printed out the pages and I couldn’t. I am sure you are trying to keep these documents manageable and short, but at times I wished that you had included more of the substance and examples from the blog posts in the within your text. 

 Regarding the “Transparency” criteria, you might be a little more specific about what you mean by “publicly available.” Do you mean that a person can find them searching the website of the charity, or that in the course of your investigations, someone at the charity eventually decided to send you the information? I wonder if these two things represent two different levels of transparency.

 In the impact analysis document, I agree with your distinction between anecdotes and representative information, but I would have liked to see more on the topic of what kind of qualitative data would satisfy your criteria (I see you linked to a blog post on this, but I would include more in the document).

 Does the content of these pages make any claims that are unsupported (when support is claimed) or unreasonable? 

 I think you provide good and careful documentation for your claims. In general I find your methodology to be systematic and well thought-out. 

 Of the three documents, I found the “guide to room for more funding analysis” to be the least strong, largely because (as you yourselves frankly point out) there is very little research on that topic for you to draw on. 

 One question that came up for me in this document was when you write about the need to focus on organizations “that appear capable of replicating activities in different regions.” From your explanations I understand why you want the organizations to be “reasonably large” (although this is a pretty fuzzy concept) but it is less clear why they should be able to span regions; I am not sure why there could not be a large organization with room for funding that operated in only one country or region.

 For the purposes for which GiveWell is using "priority programs," has a reasonable process been used to identify "priority programs?" 

 Yes, I think you’ve done the best you can with the information that’s out there. 

 You might consider discussing positive and negative externalities caused by different diseases. Your document doesn’t discuss the possibility that certain diseases have greater negative externalities, and that treating or preventing certain diseases may have greater positive externalities. Even if you conclude that we don’t have enough information to make these comparisons, you might still mention the issue. 

 Should other programs be considered "priority" that are not? Should any of GiveWell's "priority programs" not be considered "priority?" 

 This is hard for me to answer since I am not a specialist in any particular sector.

 Are the heuristics used by GiveWell reasonable for finding top contenders to meet the criteria above? Are other heuristics possible that might identify other strong charities?

 As shortcuts, they seem reasonable to me. I’m sure there are others possible but I’m not sure they would be better. I wish there were some way to get more directly at quality of management, although I haven’t come up with a heuristic for that one. Other ideas: Charities that discourage individual donors from earmarking funds (they are being honest about the fact that overhead is necessary and also that donors should not dictate the spending priorities of an organization). Charities that treat beneficiaries respectfully in their materials (ie don’t engage in poverty porn). Charities that conduct independent evaluations, even if they’re not on the website (though it’s hard to imagine why they wouldn’t be). 

 David Jinkins, July 14, 2011 About David Jinkins

 Full submission (DOC)

 Are GiveWell's criteria appropriate for individual donors (as defined above)? 

 The definition of a donor given above is very broad, as there are many reasons one might want to donate money to international aid organizations. I am not sure about what sorts of organizations fall in the category of international aid. Donors might want to give to an organization affiliated with a particular religious group or sect. They might want to promote particular values like freedom, democracy, or abstinence, or they might want recognition at home. Instead of trying to apply a one-size-fits-all program for charity evaluation, GiveWell should define its target audience more narrowly. There should be an additional bullet point containing something like: “An individual donor’s primary goal is that her donation go as far as possible to improve health and/or economic outcomes of people regardless of their location.” I think this is an operating assumption of GiveWell anyway, so it would be nice to see it stated more explicitly. If your goal in giving to an international aid organization is to improve the image of the Church of the Flying Spaghetti Monster in the United States, then you might not think any of GiveWell’s criteria for choosing a charity are relevant.

 Are GiveWell's criteria appropriately defined and defended on the pages linked to above? 

 For the most part, yes. There is a little bit too much wiggle room in some critical definitions...

 I like the way the cost effectiveness section is clear about the imprecision in the exercise...There isn’t enough of the cost-effectiveness section devoted to comparison of different types of impacts...

 GiveWell doesn’t have a page devoted to transparency.

 Does the content of these pages make any claims that are unsupported (when support is claimed) or unreasonable? 

 The pages are very well supported by sources. I was a bit frustrated by two of the issue pages linked to in the “What constitutes impact” part of the impact section...

 For the purposes for which GiveWell is using "priority programs," has a reasonable process been used to identify "priority programs?" 

 The criteria for a priority program is to be featured in both the Copenhagen Consensus and Millions Saved. Since Millions Saved is exclusively about health related programs, it is not surprising that all of GiveWell’s priority programs are health related. Some criticism might have been avoided had both of the documents used by GiveWell to identify priority programs featured more than just health related programs. Although I think that the “lower burden of proof” requirement for some health programs is justified, priority program selection is a different issue. Clearly GiveWell needs to have some easy to implement rubric to quickly evaluate a large number of charities, but it should explain why only health programs should be considered priority.

 Should other programs be considered "priority" that are not? Should any of GiveWell's "priority programs" not be considered "priority?" 

 I think there should be some objective process similar to what GiveWell has done, but with some possibility of including non-health programs. One possibility is to include programs to which J-PAL has given consistent approval. I don’t feel like I am qualified to recommend (or derecommend) specific programs, but cash grants seem to automatically fulfill GiveWell’s economic empowerment impact requirements. Maybe these could be considered.

 Are the heuristics used by GiveWell reasonable for finding top contenders to meet the criteria above? Are other heuristics possible that might identify other strong charities?

 The presence of information about the impact of a charity seems essential for GiveWell’s program. It is, of course, worrying that this information is taken from the charity’s own website, and is presumably from studies conducted by the charity itself. This is sort of like evaluating companies based on the CEO’s speeches to shareholders. I know that William Easterly has done some charity ratings, but on GiveWell’s blog it mentions that he used a similar heuristic to find charities to rate. With what is available, the heuristic GiveWell is using is acceptable, I suppose. 

 Narrowing the Field assignment Assignment description Full official assignment (DOC)

 The purpose of this assignment is to vet our process for narrowing the field from a large number of "charities to be examined" to a smaller number of "charities for futher investigation." It consists of "spot checking" different categories of charities, determining whether we have applied our criteria consistently and appropriately, and reflecting on the appropriateness of our criteria.

 We ask that the reviewer: 

Review our criteria for narrowing the field Conduct an audit of whether our criteria were applied to a random selection of the charities we list. We ask that the reviewer then select at least 20 charities, select at random, and review a mix of (a) 25% charities that we did select for further investigation (b) 50% charities that we determined did not qualify for further investigation (c) charities we deemed to be ineligible. Write thoughts on the following question: "Are there outstanding charities (as candidates for gifts from individual donors) that have been eliminated by this process? What alternative heuristics might identify outstanding charities from this large pool?"Completed assignments  Jonah Sinick, September 23 2010 About Jonah Sinick

 Full submission (DOC)

 I don’t think that GiveWell is missing out on outstanding charities on account of the weeding heuristics that GiveWell applies being unrepresentative of GiveWell’s broader criteria. The heuristics seem to me to be very reasonable.

 There’s an inherent subjectivity to applying the “Charity stands out for publishing M&E (monitoring and evaluation) reports” heuristic. Not sure what there is to be done about this. In any case, it probably doesn’t matter too much, as Holden says in Our Process: narrowing the field “we are always ready to look at charities that don’t pass these heuristics, if they meet our broader criteria.”

 As comes out in my reviews above, there are a few instances in which I feel that the heuristics may have been misapplied or applied inconsistently, but these occurrences are minor. The only possible nontrivial lapses that I perceive concern Kashf Foundation (which may warrant further investigation based on heuristic 1) and Spandana (which does in fact have a website). I recognize that Spandana may not have had a website at the time when GiveWell originally went through the “narrowing the field” process.

 Some charities publish ostensible impact reports and are nevertheless not deemed worthy of further consideration by GiveWell. In such instances, I think it would be a good idea for GiveWell to publically record why an ostensible impact report does not meet GiveWell’s standards.

  Ju Min Kim, February 2, 2011 Ju Min Kim is a postdoctoral research fellow in theoretical physics currently working in Germany. She completed this assignment as part of her volunteer work for GiveWell.

 Full submission (PDF)

 By spot-checking of charities in each categories, we confirm that overall the criteria have been applied consistently and appropriately. In particular, we find that for 2 of 5 charities in the category 1 should be reconsidered, yet, in the category 2, there was only one such charity, and in the category 3, none.

 Even though it was not the case with the charities covered by our spot-checking, we discuss that there might have been outstanding charities that have fallen outside the criteria; as a potential (minor) heuristic, we suggest transparency in usage of funds. 

 We find that applying heuristic 1 as the main filter could bring the negative side effect of the M&E sector of a charity growing too big, and it is likely needed to set some upper limit for its budget as a part of the heuristic 1. It might help to have more heuristics that can function as alternatives. Furthermore, we find that the ways of measuring impacts should be discussed more in detail.

 Charity review assignment Assignment description Full official assignment (DOC)

 The purpose of this assignment is to assess the conclusions we've come to about a particular charity that passed our heuristics (or that we reviewed in depth for another reason).

 For each iteration of this assignment, we collect all of the material relevant to a given charity (which generally includes both the review itself and any relevant blog post(s) and discussion(s) of the charity's program(s)) into a single document. We request that the reviewer read this document and write their thoughts on the following questions:

 "What do they do?" section. Does this section give you a clear picture of the charities' activities, to the point where you can picture how donations are spent? "Does it work?" section. Does this section use reasonable methods and use reasonable conclusions to assess the extent to which this charity meets the "impact" criterion laid out at http://www.givewell.org/impact-analysis ? Does the review discuss any relevant evidence base for the general kinds of programs the charity is running? (If there is a highly relevant program with a substantial evidence base, the review should link to it, and it should have been included as a separate document in your packet). Does the review competently address the question of whether there is evidence of the charity's past impact, including both "direct" evidence and evidence that the charity has executed proven programs in ways that are likely to replicate their results? Does the review explicitly raise and reasonably consider all strong "alternative hypotheses" for any empirical patterns noted as evidence of impact? (For example, if it is observed that vaccination rates rose in the area the charity worked in, one alternative hypothesis for this pattern would be that other nonprofits in the same area were working there as well.) Does the review make reasonable conclusions regarding the likelihood of future impact, considering past evidence? In assessing empirical evidence, has GiveWell used the best analytical methods available? Would other analytical methods be more helpful in reaching reasonable conclusions and predictions? (Please follow footnotes and read any Excel sheet attachments to the extent that it would help answer this question.) Does the review make a reasonable assessment of possible negative/offsetting impact, as discussed in the "impact" framework laid out at http://www.givewell.org/impact-analysis ?"What do you get for your dollar?" section. This section addresses the "cost-effectiveness" criterion laid out at http://www.givewell.org/cost-effectiveness . For reasons discussed on the cost-effectiveness page, this section aims to use external analysis as much as possible and reach a "ballpark" estimate with minimal effort. Please attempt to fully understand GiveWell's cost-effectiveness estimate, including following any footnotes and reading any Excel sheet attachments that are relevant. Are there issues with the estimates given by GiveWell (ways in which they could be substantially overstated or understated) that are not noted? Is GiveWell's conclusion the most firm that can be reached with relatively little work? Are there adjustments and/or other methods and sources that would lead to a different, and better, estimate of cost-effectiveness?"Room for more funds" section. Does this section clearly address what is known about the likely impact of additional donations? Fairness of summary. Does the summary at the top of the review accurately and fairly summarize the content of the full review?We then ask the reviewer to complete an independent assessment of the charity by examining its website, tax records, Google results, and any document attachments on the GiveWell review that seem particularly relevant to the case for its impact, cost-effectiveness and room for more funding, before answering the following: 

Is there any publicly available information that calls into question GiveWell's assertions about the charity's activities, evidence for impact, evidence for cost-effectiveness, or room for more funding? Does this independent assessment raise any important issues not discussed in the GiveWell review? Has GiveWell reached a reasonable assessment, based on the most relevant available information and best available analytical methods and data, about the extent to which this charity meets its criteria?Completed assignments  Robert Mundy, October 29, 2010: VillageReach Robert Mundy is a recent graduate from Rutgers University who completed this assignment as part of his volunteer work for GiveWell.

 Full submission (DOC)

 Bottom line:

 I examined GiveWell’s conclusion in general to see if it stood up to impact analysis principles.

 I liked that the final conclusion was optimistic, but guarded; it acknowledges that data to support VillageReach operations is not particularly representative (limited sample size, uncertainty about whether Niassa was a good comparison, problems with baseline comparison data). So I was surprised that VillageReach actually garnered a 3 star review, especially since its programs are very new and its potential to replicate its success is untested. 

 The 3 star review is well-supported by impact analysis principles, however, so the rating is internally consistent with GiveWell practices. Health programs are allowed a lower burden of proof. VillageReach’s comparison between Cabo Delgado and Niassa is based upon randomized surveys that minimize selection bias. And to address the impact criteria of systematical collection and representative overall results, I turned to the surveys and analysis GiveWell used. Specifically, I familiarized myself with the “Statistical Analysis” and “An Independent Review for VillageReach With Program and Policy Recommendations” documents associated with “Evaluation of the Project to Support PAV (Expanded Program on Immunization) in Northern Mozambique, 2001-2008.” I don’t know how to evaluate the rigor or the competency of a survey beyond some very basic markers (is its method comprehensible, was it published in a journal, can I match its conclusions to its data) but the studies seemed well-grounded in honest, accountable research. Having analyzed the review from the perspective of impact analysis, I believe that GiveWell’s 3 star rating is not an acknowledgement of a perfect charity, but instead an acknowledgement of an honest and factually sound one that benefits from the proven type of program it decided to deliver.

  Robert Mundy, December 4, 2010: Against Malaria Foundation Robert Mundy is a recent graduate from Rutgers University who completed this assignment as part of his volunteer work for GiveWell.

 Full submission (DOC)

 Bottom line:

 GiveWell’s assessment is reasonable. It does not stray toward unsubstantiated assumptions and it even points out certain key missing pieces to the story of AMF’s effectiveness. For these reasons, I can believe that a positive evaluation was not made without careful consideration. 

 I think much of my hope for the success of AMF’s activities rests on its yet-to-be-released long-term studies. Only then will I know for certain that AMF’s work adheres to GiveWell’s primary impact principle: that the charity changed lives for the better. I anticipate that this information will be included in GiveWell’s assessment as soon as it is completed and submitted for review, so I look forward to seeing your analysis of it.

  Robert Mundy, December 15, 2010: Small Enterprise Foundation Robert Mundy is a recent graduate from Rutgers University who completed this assignment as part of his volunteer work for GiveWell.

 Full submission (DOC)

 Bottom line:

 The review feels trapped between two difficult alternatives. On one hand, microfinance’s impact is not empirically supported. MFIs, like many charities, are often sketchy about their own impact and rely on anecdotal evidence, which I assume makes it even harder to weed out the good from the bad. Accordingly, one alternative might be to penalize SEF for lacking this impact data, and thus not recommend the charity. This logic suggests that no MFI deserves a recommendation, since SEF was the best among many. But on the other hand, SEF provides an amount of information that almost no other MFI does. And MFIs are clearly very attractive donor targets; billions of dollars fund SEF and similar ventures. So SEF’s openness may partially offset a solid evidence base for impact and warrant a recommendation. Even guarded recommendations will improve donor efficiency better than not having such recommendations at all. Modest improvements in “smart giving” might go a long way in such a large field.

 I have reservations about GiveWell’s conclusion, given the lack of empirical evidence at hand to support SEF impact. But I value SEF’s openness and willingness to improve. Overall, I’m happy with SEF’s Silver Medal.

  Jonah Sinick, January 1, 2011: Stop TB Partnership About Jonah Sinick

 Full submission (DOC)

 Bottom line:

 I believe that GiveWell has reached a reasonable assessment of the extent to which Stop TB meets its criteria. 

 GiveWell assesses of Stop TB as lacking evidence of a strong monitoring program for its programs to fight MDR-TB. As the GiveWell review mentions, treatment of MDR-TB is less cost-effective than treatment of TB using first-line drugs. As I mention above, the DCP figure that GiveWell quotes for the cost-effectiveness of treating MDR-TB may be overly optimistic.

 Since Stop TB reports that additional funding will be used to fight MDR-TB; depending on the cost-effectiveness of Stop TB’s efforts to treat MDR-TB (including possible prevention of a pandemic), I could imagine GiveWell’s other recommended global health charities being better bets than Stop TB at the margin. I think that it would be desirable for GiveWell take a closer look at the cost-effectiveness of Stop TB’s activities to fight MDR-TB.

 In view of the high cost-effectiveness of DOTS and the historical success of the program I find GiveWell’s #2 ranking of Stop TB reasonable when applied to Stop TB’s holistic rather than marginal output. I think that it would be desirable for GiveWell to take an explicit stand on the question of whether its rating of Stop TB “reflect(s) [GiveWell’s] opinion of organizations’ work or [GiveWell’s] opinion of whether undecided donors should give to them,” (c.f. Why Charity Ratings Don’t Work (as of now)) 

  Wendy Knight, January 25, 2011: Nurse-Family Partnership Wendy Knight completed this assignment as part of her volunteer work for GiveWell. Subsequent to completing this assignment, she joined GiveWell as a full-time staff member in February 2011.

 Full submission (DOC)

 Bottom line:

 Ooverall the methods used are sound and conclusions fair. GiveWell relies on a high standard for measuring impact, which is reflected in this review. My only caveat is where I feel further analysis could be done by GiveWell to add depth to the review. 

 Direct evidence of impact in the case of NFP comes from three randomized controlled trials (RCTs) where program participants from separate geographic regions of the country have been followed for up to 19 years post-program entry. The review does a good job of describing the results from these trials, including specific outcomes from each study. Although not explicitly stated by GiveWell, the use of RCTs means that NFP can make a strong case that any positive outcomes documented in these studies are in fact the result of the NFP intervention and not some other factor. Given this information it is reasonable to conclude that NFP meets the GiveWell criteria of providing evidence that they have changed lives for the better. In addition, because the program has been studied more than once and among a variety of populations, it can be inferred that the positive outcomes of the NFP intervention are not unique to a particular community or situation and can thus be said to be “replicable.” Due to the ability of NFP to address these two important factors (causality and ensuring findings are representative) it seems reasonable to feel optimistic, as GiveWell does, that NFP has the ability to provide positive, life-changing interventions in diverse settings. As GiveWell states in the review, “we find the evidence from these studies to provide a compelling case that the model in question improves life outcomes for children and mothers, across a variety of regions and population profiles.” 

 As the review mentions, however, there is also reason for some caution due to the fact that the research NFP relies on was carried out by one of the NFP program developers. This means some bias toward positive results may be present and/or that the researchers presence at the study sites may have influenced the facilitation of those three programs (in terms of intensity of services, enthusiasm of staff, etc.). 

 Another reason to be cautious, I believe, is that the study results show positive life changes of very different kinds across the three studies. As a donor reviewing this charity, this raises questions as to why each group seems to perform so differently. For example, the first study shows a significant reduction in incidences of child abuse and neglect 15 years post-child birth, while the second and third studies do not (granted these studies are in earlier stages of participant follow-up). Meanwhile, the second study reports that children who were part of the NFP program had better vocabulary and intellectual functioning at 6 years, but the other two studies do not show statistically significant results in this area. Even if we are to assume that each outcome is in itself important, they are vastly different effects. According to the statistically significant results listed by GiveWell, none of the same effects were seen in more than one study, drawing into question whether the program is really able to replicate itself in a consistent way. 

 With such different outcomes among the three trials, a comparison can also be made between the types of effects seen in each. For example, the last study (Denver) shows that mother’s in the NFP program enrolled children less frequently in preschool. There are many debatable assumptions regarding life outcomes built into a measure like this and I am wary to extrapolate positive benefits to the child based on those assumptions alone. In my view therefore, this is a much less favorable program outcome then a decrease in incidences of child abuse and neglect (seen in the Elmira study). 

 If we are to look at the three studies and compare results, it seems that the Elmira study (the first and only completed study, which followed the families until the children were 19) seems to have the most compelling results, both in terms of strength of findings and types of outcomes (lower child abuse and neglect and fewer arrests and convictions). Meanwhile, the other two studies reveal a limited impact on child outcomes (as mentioned above, the second study in Memphis indicates that children had better intellectual functioning at 6 years, but not at 9 and the third study in Denver indicates no clear effects for children as of the 4 year mark). As far as impact on mothers, the Memphis study outcomes are all of a much more debatable quality then the Elmira study (less time on welfare for example) and the Denver study seems to have had the greatest impact on time between births, with other discernable effects being minimal. While these studies are in their early stages still, I believe there is reason to wonder what the long-term impact of the program will be on these populations. Even where there have been small positive gains thus far, we do not know whether we will see anything near the results of the Elmira study in the long-term. This is troubling because it draws into question the idea that NFP is actually suitable and effective for a variety of populations or settings. As someone who has not seen the studies themselves, I am left to wonder whether the differences are the result of changes in location, program implementation or some other factor. If location has a significant impact, one is left to question whether the NFP model is actually replicable in all settings. If the differences are a result of variances in program implementation, then as a donor, I am most interested in funding the original high impact program. It therefore becomes important to know that NFP is doing everything it can to ensure quality and consistency across all sites. This is of critical importance because, what concerns me as a donor is how future dollars will be spent, not what the program has done in the past.

  Joshua Rosenberg, February 28, 2011: Village Enterprise Fund Joshua Rosenberg is a student at Pomona College. He completed this assignment as part of his volunteer work for GiveWell.

 Full submission (DOC)

 Bottom line:

 I do feel that GiveWell has reached a reasonable assessment based on the most relevant available information and best analytical methods and data concerning Village Enterprise Fund’s effectiveness in meeting its goals. That being said, GiveWell still seems fairly skeptical about the actual impact that VEF may be having locally. The “Silver” rating and being listed as a “Top-rated charity” may be slightly misleading for some people—they may interpret VEF as being a bonafide, highly effective charity. In actuality, VEF is being rewarded for trying hard to measure its own effectiveness. GiveWell might be more skeptical of VEF’s success internally than most people outside of the organization suspect at a glance. All things considered, though, GiveWell did the best job they could at analyzing VEF’s effectiveness and showing what reason we have to believe that VEF really is helping people.

  Kate Lang, April 29, 2011: Schistosomiasis Control Initiative Kate Lang is an MBA student at Drexel University who completed this assignment as part of her volunteer work for GiveWell.

 Full submission (DOC)

 Bottom line:

 Overall I believe that GiveWell’s assessment of SCI is thorough and reasonable given available information. There are a few areas where I would like to see greater emphasis or analysis. 

 My biggest concern after doing some independent research and reading GiveWell’s pages about NTDs [neglected tropical diseases] is what happens after the study periods and more specifically what happens when SCI exits a country and begins a new project elsewhere. I have not seen any information that indicates SCI has successfully transitioned full responsibility for mass drug administration programs to a country’s government or even sees that as a priority. The picture I see is SCI coming into a country, starting a program, analyzing its effectiveness, and then walking away—leaving the health of that country’s residents to deteriorate again. This issue is mentioned by GIveWell, but in my view it is not expanded upon or emphasized enough.

 I also make several smaller suggestions to make it easier throughout the review for the potential donor to gain a fuller understanding of the issues surrounding SCI and its work if the donor is not extremely familiar with this type of work and has not read all of the linked GiveWell pages that provide more background information. I believe it’s important to provide those clarifying points both for donors who don’t have the time to read all of the background material and also to encourage donors to take the time to read the background material by giving hints (tidbits?) as to how they apply to SCI’s effectiveness.

 Ju Min Kim, May 11, 2011: Chamroeun Ju Min Kim is a postdoctoral research fellow in theoretical physics currently working in Germany who completed this assignment as part of her volunteer work for GiveWell.

 Full submission (PDF)

 Bottom line: We find that GiveWell has reached a reasonable assessment about the extent to which this charity meets its criteria on monitoring, and we believe that this has been the main reason for GiveWell’s recommendation for Chamroeun. On the other hand, GiveWell’s assessment about the charity’s social impact appears to be overly pessimistic; accepting the relatively poor cost-effectiveness for economic empowerment programs in general, Chamroeun’s clients records and information on drop-out rates seem to show that Chamroeun meets GiveWell’s criteria on impacts more positively than GiveWell concludes. Our main concern with the review is that some of the information on which GiveWell’s assessment is based is out-dated. In particular, we could not find any solid evidence for the charity’s ability to use more funds productively since GiveWell carried out the phone conversation with it; in fact, Chamroeun seems to have become a for-profit institution in the meanwhile. We conclude that the review needs an update.

 Rachel Gross, May 20, 2011: KIPP Rachel Gross holds a master's degree in plant biology from Northwestern and is currently a graduate student. She completed this assignment as part of her volunteer work for GiveWell.

 Full submission (DOC)

 Bottom line: GiveWell’s silver medal rating for KIPP makes sense to me. KIPP’s attention to results, and evidence of very good results so far, make it a high-quality charity. The fact that it does not have a gold rating reflects the fact that certain data which would make the case stronger are not available yet. These include the results from the randomized experimental part of the Tuttle study, and data on metrics such as college graduation rates, etc that may be measurable in the future. In addition, KIPP has not yet proven the ability to reliably replicate its model, since most of the data comes from the earlier schools. 

 Another relative weakness I perceived was the limited information available about room for more funding. GiveWell lists KIPP as having “Limited” cost-effectiveness on the chart comparing U.S. charities. This seems fair enough, based on the rule on the “Guide to cost-effectiveness” page that an excellent rating requires less than $1,000 per life changed. I do wonder if that rule makes sense for this type of charity, but at the same time I did not get the sense that KIPP is excessively cost-efficient. One of the main messages in KIPP’s materials is that “there are no shortcuts” and that one of the reasons KIPP succeeds is that everyone – leaders, teachers, students, parents – works really hard. So, I wouldn’t necessarily expect that to translate to economizing of resources in comparison to other schools.

 But overall, my feeling from this review is that KIPP is a very worthy charity and that GiveWell’s review covers all the major issues.

 Lim Kwangjin, June 13, 2011: Stop TB Lim Kwangjin is a philosophy undergraduate student at New York University. He completed this assignment as part of his volunteer work for GiveWell.

 Full submission (PDF)

 Bottom line: On the whole, I think that GiveWell has reached a reasonable assessment based on the most relevant available information. Although I still have reservations regarding reliability of the GDF reports provided by the Stop TB, GiveWell rightly infers from the limited information that the GDF’s first-line drug program has been effective in that its cure rate and treatment success rate are fairly high even in high-burden areas. And, the review provides a comprehensive and critical examination of the monitoring program and rightly concludes that the monitoring program for the first-line drugs is indeed reliable, although it is not without shortcomings. On the other hand, as GiveWell believes that a significant portion of additional donations will be spent on the MDR-TB program, for which GiveWell hasn’t seen evidence of a strong monitoring process, the review must be updated as soon as relevant information becomes available.

 Considering that there is a relatively low burden of proof for charities like the Stop TB that employs a proven cost-effective strategy, I think that the Stop TB’s ranking as a Silver Medal organization is well deserved. Their strategy, DOTS, has been proven to work effectively and the evidence in the review strongly suggests that the Stop TB’s self-monitoring program ensures proper implementation of the strategy. Moreover, throughout the review, the Stop TB is shown to be cooperative and responsive to GiveWell’s requests for necessary information. Such cooperation could be a testament to the organization’s transparency and honesty. In sum, that the organization employs a proven cost-effective strategy, tries to ensure its proper implementation, and is responsive and honest in its information sharing is a solid basis for GiveWell’s granting a silver medal.

 One question that the review could further address is why the Stop TB does not want to publicly disclose information they provided to GiveWell. As I am certain that the Stop TB does have a plausible reason not to do so, noting that reason in the review could help resolve any reservations one might have, especially when it was shown that some of their reports relied on information from previous ones without updating.

 Rachel Gross, September 5, 2011: Invest in Kids Rachel Gross holds a master's degree in plant biology from Northwestern and is currently a graduate student. She completed this assignment as part of her volunteer work for GiveWell.

 Full submission (DOC)

 Bottom line: I think GiveWell’s designation of Invest In Kids as a silver medal organization is very appropriate. The organization’s commitment to implementing proven programs and to evaluating itself continuously makes it an organization with standards similar to GiveWell. I think this review summarizes all the negative and positive issues relating to how effectively a donation to this charity would be used. I assume that as the OMNI Institute produces a new report each year, there will be opportunities to periodically update and refine the assessment.

 Ju Min Kim, November 12, 2011: VillageReach Ju Min Kim is a postdoctoral research fellow in theoretical physics currently working in Germany who completed this assignment as part of her volunteer work for GiveWell.

 Full submission (PDF)

 Bottom line: Based on GiveWell's standard review on VillageReach, we believe that GiveWell has reached a reasonable assessment about the extent to which this organization meets its criteria. In particular, we also find the evidence of impacts, transparency and self-evaluation excellent. The cost-effectiveness seems to be slightly overestimated, considering that VillageReach is an organization working to improve only the logistics among various factors that affect vaccination. However, we believe that even being more conservative, the conclusion will not deviate much.

 Finding the Best Charity assignment Assignment description Full official assignment (DOC)

 The purpose of this assignment is to gauge the usefulness of GiveWell's research in a real-world context: deciding what charity to give to in a limited amount of time.

 We ask the reviewer to spend a set number of hours choosing the charity s/he would most like to support with a donation; in some cases we may actually offer to make the donation, as a form of "compensation" for the reviewer's time.

 We require that at least half of the funds are allocated to a charity/charities to which the reviewer has no personal connection. This requirement is in keeping with the assignment's goal of testing the usefulness of GiveWell's research.

 We ask that the reviewer consider the research available at GiveWell as a resource for helping you make your decision. However, we do not require that the reviewer spend any particular amount of time reviewing it, and we also refer the reviewer to other donor resources including Charity Navigator, GuideStar, Great Nonprofits, Philanthropedia, and Giving What We Can. 

 The reviewer is asked to write thoughts on the following questions after s/he has made his/her decision: 

What charity or charities did you pick, and how did you allocate the funds between them? Why? Which of these charities do you have personal connections to? What is the nature of these connections? How confident are you that your donation is a "good bet" to have impact? Please give a brief overview of the process you followed to make your decision, including websites visited & phone calls made. What resources played a major role in your decision? What do you see as the strengths, weaknesses, and area for improvement of the different resources (GiveWell, Charity Navigator, etc.) you used?Completed assignments  Stephanie Wykstra, July 21 2010 About Stephanie Wykstra

 Full submission (ZIP)

 What charity or charities did you pick, and how did you allocate the funds between them? Why?

 My top choice charity is: Stop Tuberculosis Partnership. I would give the full donation to this group.

 Which of these charities do you have personal connections to? What is the nature of these connections?

 I do not have a personal connection to this organization.

 How confident are you that your donation is a "good bet" to have impact?

 I am relatively confident that my donation is a good bet to have impact.

 Please give a brief overview of the process you followed to make your decision, including websites visited & phone calls made.

 1. Started by investigating my goal, including my beliefs about what should take priority (i.e., what I should be aiming for as a goal).

 What I used: I relied on my own philosophical reflections.

 What I learned:  Reflections on my beliefs allowed me to arrive at the aim of: working to meet basic human needs (water, food, shelter, crucial medical care) where they are lacking (i.e., the developing world).

 2. Among the basic needs, I researched efforts to assist with / meet the need for water, shelter and food and medical care.

 What I used:  mostly GiveWell’s website and in particular, the pages on developing-world education, the pages and blog post on water, and the pages on economic empowerment and microfinance. I found these pages very helpful partly because of the research (and links) they are based on. Since I am familiar with the Poverty Action Lab and other research centers and believe that GiveWell stays on top of their research, I feel that I’m able to trust the GiveWell is taking into account the relevant studies that are available.

 What I learned:  I ruled out the causes other than medical interventions, each for one (or both) of the following reasons: 1. Lack of significant provable success of various measures in addressing the problem, 2. The problem did not seem to me as serious as disease in the developing world. See Research notes (DOC) for more specifics.

 3. Having chosen international medical interventions as a focus, I attempted to investigate which of the organizations recommended by GiveWell was the best. 

 What I used:  mostly GiveWell’s pages on these organizations, but also Giving What We Can and a few other websites (see Notes, “notes on websites,” XLS). 

 What I learned:  I had many considerations in mind when trying to select an organization (see “Research Notes”). I found that it was extremely difficult to tell – among the ones listed in my selection page (see Notes, “medical” XLS) – which would be the best to give to. 

I had some concerns (see “Concerns” column) but really couldn’t tell, at least not without a lot more research than I planned to do in 10-12 hours, which concerns were legitimate and which were not. For example: I had some ideas about some selection criteria which would allow me to choose among GiveWell’s top-rated. For example, I didn’t choose PSI or Against Malaria Foundation because of my criterion that the less recipient compliance is essential for the treatment to work, the better (since I had in mind that many people may not comply because of lack of knowledge and understanding of medical risks and causes (of HIV and malaria in particular), especially in areas with little education available). However, I don’t know if my criterion was a good one to use, given that the organization might be highly successful in spite of some level of non-compliance.

 In summary: being able to discern which organization to give to, among international health organizations recommended by GiveWell, is something that I am skeptical about. Happily, if GiveWell’s research is reliable (as it seems to be), this shouldn’t be a problem for donors. They can just choose whichever top-recommended organization they like, and presumably, their donation will go to an effective group working on an important problem. What resources played a major role in your decision?

 I mostly relied on GiveWell and to an extent, on Giving What We Can (which largely relies on GiveWell). I did look at other websites, but found them less helpful. (Guidestar relies largely on GiveWell, it seems, and links to it often; Charity Navigator does not address most of the questions that I ask when considering where to give. I am not sure how reliable Philanthropedia is, and in any case, it doesn’t address the area I was researching.

 What do you see as the strengths, weaknesses, and area for improvement of the different resources (GiveWell, Charity Navigator, etc.) you used?

 Most of the things that I noticed on the GiveWell sites that I used were smaller things that I can speak to another time. I did find a few bigger things, however:

 Possible areas for improvement/weaknesses: 

Some donors (like me) may wonder not just which causes can efficiently help people, but instead may begin with the question: what should our goal be? If the answer is (as it’s likely to be for donors), “Improve the world,” then the question is: what does this best? Answering this question obviously requires some reflection on what the goal is. Let’s say it’s “helping people” (as opposed to helping animals or plants). In that case, what helps people the most? I answered this on my own (See Preliminary Notes) and noticed that I was making a lot of assumptions along the way. In making recommendations, GiveWell is also making assumptions about its priorities in helping people. And laying out those assumptions somewhere on the site might be good (I think they are mentioned on the blog in various places and perhaps elsewhere, but not compiled.) Compiling them might be a good idea, though perhaps not (it might alienate some donors who do not share the assumptions, to see them listed explicitly – I’d have to think more about this point to come to a considered view about it.)If someone wants to compare the top-rated and recommended charities in some way other than just looking at the stars given and the chart here http://www.givewell.org/charities/top-charities, it is extremely difficult. I.e., I tried to consider the information provided in the summaries, and had no idea which of the charities was better or worse (see my research summary, step 3, above). There are too many factors to weigh. So in the end, the number of stars GiveWell assigns is probably the biggest factors that will lead those who want to give to the best organizations to their decision. I did have a few questions about GiveWell’s decisions about how many stars to assign (in the case of the top 12). Why would U.S. charities figure at all on the top 12 list? They are clearly not on a par in terms of meeting fundamental needs. Perhaps they are high-ranking in terms of non-profit work in the U.S. But putting them in the same list makes them seem comparable in terms of how much good they do (though this might not have been the intention): I don’t think saving millions of lives internationally can be compared on a par to creating more educational opportunity in the U.S., assuming that all lives are equally valuable (which I do, and I think GiveWell does also). So perhaps these should be in a different list altogether (U.S. versus International), on the recommended charities page?

 I have some questions about the criteria used, but I’ll only mention briefly: Why is transparency to the public a criterion for the ratings? I can see how GiveWell would need the information and I am completely in favor of non-profit culture changing to be research and measurable-results focused. But since GiveWell’s aim is to direct the donors to effective organizations, it doesn’t seem to me that donors need to be able to vet the organization themselves (which would, after all, be extremely time-consuming and just what the vast majority of even GiveWell users would not want to do, if possible). I think the rationale may be that donors shouldn’t have to trust GiveWell, because they can see the research for themselves. However, this would be the case as long as GiveWell is transparent about its own gathered research and sources (and it is, I believe). It doesn’t require further that the organization has to be (to the donors themselves). My concern here is that this is not obviously necessary at all, and might just require a lot of extra website work and updating for the organizations.Strengths: Overall, this project just made it clear how incredibly useful GiveWell’s research is to donors (hopefully a growing number) who would like to make good decisions in donating. It is extremely challenging and maybe impossible to make informed decisions without a source of reliable information, and GiveWell makes this possible.

  Robert Mundy, October 11, 2010 Robert Mundy is a recent graduate from Rutgers University who completed this assignment as part of his volunteer work for GiveWell.

 Full submission (DOC)

 What charity or charities did you pick, and how did you allocate the funds between them? Why?

 I decided to donate 25% to charities I already knew, and 75% to charities I had never heard of. I wanted to use GiveWell and other tools more extensively than I might if more of my money was going to charities I was already familiar with. 

 Partners in Health (20%)

 Conclusion: I probably made the right choice by donating to Partners in Health, and would like to continue my support. I value PIH’s commitment to a job that many didn’t think possible—providing comprehensive healthcare to a region without even the most basic centrally controlled healthcare delivery system. Despite losing staff members and facilities during the recent Haiti earthquake, their work continues to expand. Though I could probably find a more cost-effective charity, I choose to allocate 20% to PIH as recognition of their dedication. I explain more about this sentiment in my larger conclusion at the end of this assignment.

 Charity:Water (5%)

 Conclusion: I would allocate 5% of my donation budget to Charity: Water. This 5% allows me to track the correspondence between Charity:Water and donors (by virtue of me donating at all, and being part of their system) to confirm that the website’s efforts to be transparent come to fruition. The nonprofit’s multimedia approach is still very appealing to me—revealing the individuals being aided by donations in high definition video is a rarity, and might produce a more meaningful connection between donor and recipient that could encourage consistent, long-lasting donations on a large scale. I value that aspect of Charity: Water’s approach, even if I can’t fairly support the claims that the nonprofit makes about its work. 

 Village Reach (25%)

 VillageReach is a strong candidate. Since my sole criteria is cost-effectiveness, and Village Reach appears to be both highly cost-effective and accountable enough to remain so in the future when my donation takes effect, it definitely warrants a part of my remaining 75%. In fact, it would be tempting to just allocate all of the 75%, since I can’t find any other satisfactory online listing of developing world-focused charities ranked by cost-effectiveness, which makes GiveWell the authority on the issue. Since VillageReach is ranked 1st on your list, logic might dictate I should dump my money on my best chance. But relying solely on a single organization’s analysis is somewhat foolhardy, especially since cost-effectiveness is difficult to determine and is largely region/circumstance-specific.

 Stop TB Partnership (25%)

 Conclusion: Stop TB Partnership is immensely cost-effective and seems willing and able to provide me the information I would need to confirm that my donation is helping. Interestingly, the website contains much more information about TB and why we should be working to stop it at all than a defense of the Stop TB model in specific. This might be because the group is effectively a branch of a supranational group the (UN) that probably envisions health advocacy to be as crucial to healthy people as health implementation. Regardless, my money would be well spent by donating to Stop TB Partnership.

 Against Malaria Foundation (25%)

 Conclusion: Against Malaria Foundation offers a straightforward interface for viewing malaria prevention projects around the world, which was a big plus. Doing so warrant at least a certain minimum of accountability (you can’t show what you haven’t done) and also allows me to track the progress of my donation. Cost-effectiveness was my biggest deciding factor, and since bednets can save a life for a (high-end) estimated $1000, my decision was an easy one.

 Which of these charities do you have personal connections to? What is the nature of these connections?

 Personal Connection to PIH: I read Tracy Kidder’s Mountains Beyond Mountains and learned about PIH through the book. I researched the organization and have been donating fairly regularly since.

 How confident are you that your donation is a "good bet" to have impact?

 I am reasonably confident. While I don’t think cost-effectiveness in terms of lives saved is the robust equation I wish it was, and hope it some day will be, I can rely on it enough to expect my money will go where it should needs to. I feel significantly more confident as a result of using GiveWell than I would have without using it. There simply isn’t enough free, available information online for me to make a calculated decision to support one charity over another. 

 What resources played a major role in your decision?

 I examined each resource you gave me. I was previously familiar with Guidestar and Charity Navigator so I focused most of my time on the newer resources. I decided to rely on Charity Navigator and GiveWell for my charity decisions, while using Guidestar and Givewhatwecan.org to supplement my research.

 Charity Navigator: offers helpful guidelines for measuring transparency and accountability. It provides financial information in a clear, identifiable way and provides ratings that help me gauge an organization’s track record success and its potential for using my donation effectively.

 GiveWell: well-suited to my interests: most of the charities it reviews possess mission statements that are similar to my own donation goals: saving lives and drastically improving livelihoods at the cheapest cost. I don’t have to wade through charities that are well-rated but don’t suit my needs, as I might on Charity Navigator. 

 Givewhatwecan.org: compares health solutions by cost-effectiveness. Not enough content to be used exclusively, but was helpful for Stop TB Partnership.http://www.givinhatwecan.org/resources/health.php

 What do you see as the strengths, weaknesses, and area for improvement of the different resources (GiveWell, Charity Navigator, etc.) you used?

 GiveWell will prove most useful to donors that want primarily want their money to go the furthest toward saving lives. I don’t think GiveWell receives significant competition from other sources because even alternatives that offer more listings (Charity Navigator and GuideStar), don’t simultaneously 1) focus primarily on cost effectiveness and 2) primarily target the most life-threatening problems in the way that GiveWell does.

 GiveWell’s most pressing weakness is its lack of content. As a new organization with a small staff, this isn’t a testament to lack of effort, merely a lack of resources to provide something akin to a comprehensive analysis of the world’s top charities that I want. In time, this might be possible. For now, I feel I have several options when donating my money, and that is satisfactory.

 GiveWell’s greatest strength, and probably the biggest reason I became interested in volunteering in the first place, is the website’s focus on accountability. Citations, descriptions, explanations of failures, and admittances of doubt are all key parts to a charity rating source that I can trust. When I need to decide where to send my money, I need to trust implicitly that the organization functioning as my proxy maintains the same standards that I do. Without this trust, I cannot fully justify using a charity ranking as a substitute for my own knowledge. GiveWell earns that trust by accepting that it is flawed, unfinished, but well-referenced.

 One suggestion involves your star ranking system. Ranking charities by percentile might better describe the system you use gauge charity effectiveness. Among other product reviews (movies, restaurants, books, etc.), 2 out of 3 stars would indicate an average or slightly above average ranking—and since people using GiveWell are probably used to that standard, a 2 out of 3 star ranking probably won’t seem very impressive when they visit your site for the first time. If a casual reader didn’t read the note beneath each ranking, they might get the wrong idea about the charities you analyze. A percentile (2 out of 3 stars = 97th percentile) might better characterize your intent.

  Timothy Moreland, December 17, 2010 Timothy Moreland is a recent graduate from University of Pittsburgh who completed this assignment as part of his volunteer work for GiveWell.

 Full submission (DOC)

 What charity or charities did you pick, and how did you allocate the funds between them? Why?

 I chose Village Reach (35% of the funds), Against Malaria Foundation (35%), PSI (20%), and Stop TB (10%). In Philanthropedia’s Guide To Better Giving, they suggested to support multiple nonprofits, as this would allow for different approaches to be attempted. As well, I wanted to avoid diminishing returns on my donation, as each additional dollar could have less of a marginal impact. For these reasons, I wanted to split the funds between the charities I chose. Also, I allocated funds based on my perceived likelihood that the money would have an impact. Since Stop TB Partnership did not have as clear of a need for funds and an unproven evaluation process, they receive a smaller allocation. On the other hand, Village Reach’s focus on programs that have historically made a strong impact, as well as their excellent monitoring and evaluation, earned them a greater share of the funds.

 Which of these charities do you have personal connections to? What is the nature of these connections?

 None of these charities.

 How confident are you that your donation is a "good bet" to have impact?

 Highly confident, as all evidence points toward Village Reach and AMF being a very good bet to have an impact (50% of the donation) and the others have a significant likelihood of making an impact, based on GiveWell and Giving What We Can evaluations. Since I allocated money to those charities that displayed strong evidence of cost-effectiveness, potential for impact, and a funding gap, the dollars should certainly make a real-world impact.

 What resources played a major role in your decision?

 GiveWell’s listing of the Top Charities made the biggest impact on my final choices. None of the other resources offered as clear of a ranking system for who to donate toward, with the possible exception of Giving What We Can. However, GWWC is limited to health nonprofits at this time.

 What do you see as the strengths, weaknesses, and area for improvement of the different resources (GiveWell, Charity Navigator, etc.) you used?

 Charity Navigator Strength: Large database of charities Weakness: There are several perfect star ratings, which makes it very difficult to make a quick decision on which charity to support. Area for Improvement: They should do more in offering guidance on how a donor should allocate his money.GuideStar Strength: Provides unique information, such as funding needs and specifics on programs within organizations Weakness: Reviews are done by public, which could bring into question credibility of reviewsGreat Nonprofits Strength: Provide empirical results of some charities’ impactWeakness: Reviews are done by public, which could bring into question credibility of reviewsPhilanthropedia Strength: Rankings are based on opinion of diverse group of relevant experts Weakness: Microfinance is the only international cause where they currently rank nonprofits Areas for Improvement: Methodology of final rankings of nonprofits could be more explicit and transparentGiving What We Can Strength: Clear recommendations of best charities to donate to Weakness: Focused solely on Health nonprofits at this time Areas for Improvement: Create more in-depth reports on specific charities and make reasons for rankings more explicitGiveWell Strengths: Detailed reports of charities, clear methodology Weaknesses: Limited number of full charity evaluations at this time Areas for Improvement: Expanded rankings within impact program areas Tobias Pfutze, March 2, 2011 About Tobias Pfutze

 Full submission (DOC)

 Follow up conversation with Professor Pfutze (DOC)

 What charity or charities did you pick, and how did you allocate the funds between them? I decided to donate the entire fund to Living Goods.

 Which of these charities do you have personal connections to? What is the nature of these connections? I do not have any personal connection to the charity.

 How confident are you that your donation is a "good bet" to have impact? As I discuss in more detail in the research notes, I believe that one should focus on the expected value of a charity investment. In the case of Living Goods a rigorous evaluation is being carried out, but results are still pending. But the potential impact from the project is, in my opinion, huge. I was especially drawn to Living Goods as it is the only charity that focuses on creating a market for basic health goods that promises to be self-sustaining in the long run.

 Please give a brief overview of the process you followed to make your decision, including websites visited & phone calls made. I conducted my entire search online, visiting Givewell’s and other charity evaluation websites, as well as, charities’ own web pages. At one point, I wrote one short email to Givewell for clarification.

 What resources played a major role in your decision? The most important resource consulted was Givewell’s web page.

 What do you see as the strengths, weaknesses, and area for improvement of the different resources (GiveWell, Charity Navigator, etc.) you used? I provided a more detailed description of the different web sites in my research notes. In short, I found Givewell to be by far the most helpful website. It is the only one that provided an in-depth discussion on its recommended charities with a strong focus on proven results (which no other web page does). That said, I think it would benefit from providing more information, such as basic financial data, on the charities that are not in the top group. If that information is missing, some of the aggregator websites (e.g. charitynavigator) can be of help. In general, in order for the other websites to be of any use, one needs to be looking for a specific charity already.

 Rachael Stephens, July 21, 2011 Rachael Stephens is a 2011 graduate of Bryn Mawr College. She completed this assignment as part of her volunteer work for GiveWell.

 Full submission (DOC)

 Charities selected, connections, and allocation of funds: I selected Small Enterprise Foundation* (SEF) and Village Enterprise Fund (VEF) as two charities to donate to, with Village Enterprise Fund receiving $1,000 and the Small Enterprise Foundation receiving $1,500 of the total $2,500 [hypothetically] available for donation. 

 I selected these two nonprofits because a) I was approaching the selection process with a certain set of interests/areas of concern – economic empowerment and development – as I’m sure several donors using the site might have coming in to the process; and b) I felt confident that, despite some of their flaws, they seem to be doing the best to monitor their activity, which is the first and very necessary step in developing feasible solutions to those problems. I felt confident in this because of GiveWell’s research and rigorous criteria here as well as what I was able to easily find out about each organization via their own websites. I also found the fact that they both include training programs and mentorships led or taught by locals to be critical, as the effects of such training or other education is much more empowering, with longer-lasting effects than simply a loan or grant alone. An additional benefit of selecting these two was that they did not approach the problem in the same way, so it was different than donating to two similar microfinance programs, for example.

 I do not have personal connections to either of these organizations.

 Unsolicited feedback Gilles Corcos, November 1, 2012 Gilles Corcos is a Professor Emeritus in the Mechanical Engineering Department at UC Berkeley and a founding member of Agua Para La Vida, an NGO that works on water and sanitation issues in rural Nicaragua. He emailed us the following feedback, and gave us permission to share it publicly: 

Letter (DOCX) Attachment (DOC; a form used to evaluate APLV water projects; in Spanish) FIECON, January 2018 In January 2018, FIECON, a health economics consultancy, completed an independent audit of our cost-effectiveness analysis of the Against Malaria Foundation on a volunteer basis. We have not had capacity to fully respond to this report. We note that there are differences in methodologies and inputs from GiveWell's approach, but our understanding is that the report's bottom line is broadly consistent with our conclusions. FIECON has given us permission to share their audit publicly:

 Cost-effectiveness Model Technical Report: Long-lasting insecticide-nets for the prevention of malaria in rural and urban malarious areas LLIN cost-effectiveness model The executive summary of the report follows:

 GiveWell is a non-profit organisation, dedicated to finding outstanding giving opportunities for donors. As of December 2017, GiveWell recommends the Against Malaria Foundation (AMF) as a top rated charity, which provides long-lasting insecticide-treated nets (LLINs) for the protection against malaria in developing countries. FIECON is a health economics consultancy, and has conducted an independent audit to evaluate the cost-effectiveness of LLINs for the prevention of malaria in adults and children from rural and urban malarious areas.

  A cost-effectiveness model (CEM) has been developed to compare the costs and quality adjusted life years (QALYs) of preventing malaria with LLINs, untreated nets (UTNs) and no nets (NNs). Three populations enter the model; children aged less than 5 years old, children aged between 5 and 18 years old, and adults and children of any age. The model adopts a UK net provider perspective, i.e. all costs borne from the donor (AMF). A 3 year time horizon with yearly cycles is used, and a discount rate of costs and outcomes is applied at 3% per annum.

  A Markov model structure is adopted with health states based on an individual’s malaria diagnosis. Health states are defined as ‘Alive and healthy’ and ‘Alive with malaria’, and ‘Dead’. The probability of transitioning between states is based on published literature obtained from a systematic literature review (SLR) and targeted literature searches. Cost categories included in the model are acquisition costs, implementation costs and retreatment costs. Utilities are based on published literature. Sensitivity analyses are performed to explore the level of uncertainty in the model results.

  The base case results found that LLINs are cost-effective in comparison to UTNs and NNs with an incremental cost-effectiveness ratio (ICER) below £1,000 per QALY in all populations. Despite the fact that the analysis uses an entirely different model structure compared to GiveWell, a variety of different sources to derive clinical effects, and a broader population scope, it is reassuring to observe that the cost-effectiveness results calculated by GiveWell are similar to those calculated in this analysis. It can be confidently concluded that LLINs for the prevention of malaria are a cost-effective intervention in all populations where malaria is at high risk.

                           Home Contact Stay updated FAQ For Charities Site map Privacy Policy       Follow Us: Facebook Twitter RSS   Subscribe to email updates:   GiveWell, aka The Clear Fund (a tax-exempt 501(c)(3) public charity), was founded in 2007. We serve donors across the Globe; GiveWell's donors are based primarily in the United States, United Kingdom, Australia, Germany, and Canada. This work is licensed under a Creative Commons Attribution-Noncommercial-Share alike 3.0 United States License          try { clicky.init(78566); }catch(e){} 

   window.intercomSettings = { "app_id": "you5eu27" };  (function(){var w=window;var ic=w.Intercom;if(typeof ic==="function"){ic('reattach_activator');ic('update',intercomSettings);}else{var d=document;var i=function(){i.c(arguments)};i.q=[];i.c=function(args){i.q.push(args)};w.Intercom=i;function l(){var s=d.createElement('script');s.type='text/javascript';s.async=true;s.src='https://web.archive.org./web/20190428104243/https://widget.intercom.io/widget/APP_ID';var x=d.getElementsByTagName('script')[0];x.parentNode.insertBefore(s,x);}if(w.attachEvent){w.attachEvent('onload',l);}else{w.addEventListener('load',l,false);}}})()