Education in Developing Countries | GiveWell               (function(w,d,s,l,i){w[l]=w[l]||[];w[l].push({'gtm.start':new Date().getTime(),event:'gtm.js'});var f=d.getElementsByTagName(s)[0];var j=d.createElement(s);var dl=l!='dataLayer'?'&l='+l:'';j.type='text/javascript';j.src='https://web.archive.org./web/20201024203016/https://www.googletagmanager.com/gtm.js?id='+i+dl+'';j.async=true;f.parentNode.insertBefore(j,f);})(window,document,'script','dataLayer','GTM-MKLT9TL');     jQuery.extend(Drupal.settings, {"basePath":"\/","pathPrefix":"","ajaxPageState":{"theme":"gw_basic","theme_token":"gq4tXnPImZFWPnMmi_0mCyFx8AERHik5moTp8c55hVU","js":{"https:\/\/web.archive.org.\/web\/20201024203016\/https:\/\/unpkg.com\/@popperjs\/core@2":1,"https:\/\/web.archive.org.\/web\/20201024203016\/https:\/\/unpkg.com\/tippy.js@6":1,"0":1,"sites\/all\/modules\/contrib\/jquery_update\/replace\/jquery\/1.10\/jquery.min.js":1,"misc\/jquery-extend-3.4.0.js":1,"misc\/jquery-html-prefilter-3.5.0-backport.js":1,"misc\/jquery.once.js":1,"misc\/drupal.js":1,"sites\/all\/modules\/contrib\/jquery_update\/replace\/ui\/ui\/minified\/jquery.ui.core.min.js":1,"sites\/all\/modules\/contrib\/jquery_update\/replace\/ui\/ui\/minified\/jquery.ui.widget.min.js":1,"sites\/all\/modules\/contrib\/jquery_update\/replace\/ui\/external\/jquery.cookie.js":1,"sites\/all\/modules\/contrib\/jquery_update\/replace\/ui\/ui\/minified\/jquery.ui.accordion.min.js":1,"sites\/all\/modules\/contrib\/resp_img\/resp_img.js":1,"sites\/all\/modules\/contrib\/tableofcontents\/js\/jquery.scrollTo-min.js":1,"sites\/all\/modules\/contrib\/tableofcontents\/js\/jquery.localscroll-min.js":1,"sites\/all\/modules\/contrib\/tableofcontents\/js\/tableofcontents.js":1,"sites\/all\/modules\/custom\/givewell_js\/js\/intercom.js":1,"sites\/all\/modules\/custom\/givewell_add_querystring\/js\/givewell_add_querystring.js":1,"sites\/all\/themes\/gw_basic\/scripts\/lib\/modernizr.custom.74085.js":1,"sites\/all\/themes\/gw_basic\/scripts\/lib\/mean-menu\/jquery.meanmenu.min.js":1,"sites\/all\/themes\/gw_basic\/scripts\/lib\/jquery-migrate-1.0.0.js":1,"sites\/all\/themes\/gw_basic\/scripts\/lib\/jquery-validation\/dist\/jquery.validate.min.js":1,"sites\/all\/themes\/gw_basic\/scripts\/lib\/multiselect\/jquery.multiselect.min.js":1,"sites\/all\/themes\/gw_basic\/scripts\/lib\/dropkick.js":1,"sites\/all\/themes\/gw_basic\/scripts\/lib\/jquery.pep.js":1,"sites\/all\/themes\/gw_basic\/scripts\/main.js":1},"css":{"modules\/system\/system.base.css":1,"modules\/system\/system.menus.css":1,"modules\/system\/system.messages.css":1,"modules\/system\/system.theme.css":1,"misc\/ui\/jquery.ui.core.css":1,"misc\/ui\/jquery.ui.theme.css":1,"misc\/ui\/jquery.ui.accordion.css":1,"modules\/aggregator\/aggregator.css":1,"modules\/comment\/comment.css":1,"sites\/all\/modules\/contrib\/date\/date_api\/date.css":1,"modules\/field\/theme\/field.css":1,"sites\/all\/modules\/contrib\/footnotes\/footnotes.css":1,"modules\/node\/node.css":1,"modules\/search\/search.css":1,"modules\/user\/user.css":1,"sites\/all\/modules\/contrib\/views\/css\/views.css":1,"sites\/all\/modules\/contrib\/ckeditor\/css\/ckeditor.css":1,"sites\/all\/modules\/contrib\/ctools\/css\/ctools.css":1,"sites\/all\/modules\/contrib\/taxonomy_access\/taxonomy_access.css":1,"sites\/all\/modules\/contrib\/content_type_extras\/css\/content_type_extras.css":1,"sites\/all\/modules\/contrib\/tableofcontents\/tableofcontents.css":1,"sites\/all\/modules\/contrib\/print\/print_ui\/css\/print_ui.theme.css":1,"sites\/all\/themes\/gw_basic\/styles\/css\/global\/normalize.css":1,"sites\/all\/themes\/gw_basic\/styles\/css\/global\/gw-fonts.css":1,"sites\/all\/themes\/gw_basic\/styles\/css\/global\/html.css":1,"sites\/all\/themes\/gw_basic\/styles\/css\/global\/global.css":1,"sites\/all\/themes\/gw_basic\/styles\/css\/global\/meanmenu.css":1,"sites\/all\/themes\/gw_basic\/styles\/css\/global\/page-layout.css":1,"sites\/all\/themes\/gw_basic\/styles\/css\/global\/html-colors.css":1,"sites\/all\/themes\/gw_basic\/styles\/css\/pages\/pages.css":1,"sites\/all\/themes\/gw_basic\/styles\/css\/pages\/new-home.css":1,"sites\/all\/themes\/gw_basic\/styles\/css\/pages\/new-top-charities.css":1,"sites\/all\/themes\/gw_basic\/styles\/sitesearch.css":1,"sites\/all\/themes\/gw_basic\/styles\/formstack.css":1,"sites\/all\/themes\/gw_basic\/styles\/css\/global\/admin.css":1}},"respImg":{"default_suffix":"","current_suffix":false,"forceRedirect":false,"forceResize":false,"reloadOnResize":false,"useDevicePixelRatio":false,"suffixes":{"":1}},"tableofcontents":{"collapse":false,"scroll":true},"urlIsAjaxTrusted":{"\/international\/technical\/programs\/education":true}});       Jump to Navigation             GiveWell        Giving EffectivelyGiving 101 Your Donation Can Change Someone's Life The Wrong Donation Can Accomplish Nothing Your Dollar Goes Further Overseas Your Dollar Goes Further When You Fund the Right Program Quick Start Guide  How We WorkWho We Are Process Criteria Transparency Evaluations of GiveWell Research FAQ Core Competencies  Top CharitiesMalaria Consortium Against Malaria Foundation Helen Keller International Deworm the World Initiative SCI Foundation Sightsavers The END Fund GiveDirectly Standout Charities  ResearchCharity Site Visits Notes from Research Conversations Intervention ReportsAlbendazole and Ivermectin to Control Lymphatic Filariasis Alive & Thrive Antiretroviral Therapy to Treat HIV/AIDS Bednet Distribution Breastfeeding Promotion Programs Bridges to Prosperity Cash transfers Cataract Surgery Clean Cookstoves Combination Deworming Community-Based Intervention Packages Community-Based Management of Acute Malnutrition Conditional Subsidies for Seasonal Labor Migration in Northern Bangladesh Condom Promotion and Distribution to Prevent HIV/AIDS Distribution of Eyeglasses in Developing Countries Early Childhood Psychosocial Stimulation Education in Developing Countries Eyeglasses to Improve Workers' Manual Dexterity Immunization Immunization to Prevent Maternal and Neonatal Tetanus Integrated Community Case Management (iCCM) Intermittent preventive treatment of malaria during pregnancy Interventions to Promote Handwashing Iron Fortification Iron Supplementation for School-Age Children Ivermectin to Control Onchocerciasis Mass Distribution of Azithromycin Meningitis A Immunization Non-therapeutic Zinc Supplementation Oral Pre-exposure Prophylaxis (PrEP) for HIV Oral Rehydration Solution Prevention of Mother-to-Child Transmission of HIV Probiotic Supplementation for Preterm Newborns Salt Iodization Sayana® Press Seasonal Malaria Chemoprevention SMS Reminders for Vaccination Soft Skills Certification Supplementary Immunization Activities to Prevent Measles Surgery to Repair Obstetric Fistula Syphilis Screening and Treatment During Pregnancy Therapeutic Zinc Supplementation Trachoma Control Treatment of Malaria Tuberculosis Case Finding and Treatment Vitamin A Supplementation Voluntary Medical Male Circumcision (VMMC) HPV Vaccination to Prevent Cervical Cancer IPTi Water Quality  Cost-Effectiveness Analyses Other Charity Reviews GiveWell Incubation Grants  Our Mistakes AboutAbout GiveWell Progress to Date Our Story Our People Official Records Reputation Impact Frequently Asked Questions Donate Contact Us Jobs  UpdatesBlog RSS Feeds Stay Updated           Enter search terms here.   This search returns results from both GiveWell's main site and from the GiveWell Blog. 

  Search form  Search             DonateDonate

            You are hereHome » Research » Intervention Reports » Education in Developing Countries Education in Developing Countries     FacebookTwitter>Print>Email                    Summary Focus of this page: This page discusses our current view of the evidence for a wide range of programs and interventions that aim to improve education in developing countries. These include demand-side interventions that lower the cost of schooling or increase its (perceived) returns, provision of school inputs, pedagogy interventions, and governance reforms. We focus mainly on interventions aimed at improving primary and secondary education but consider vocational training interventions briefly. We have not yet completed a report on early childhood (pre-school) interventions. On this page, we focus on evidence from experimental study designs. What evidence do we have of these programs' effectiveness? This page focuses on the direct evidence for the effect of education interventions on outcomes (e.g. improved earnings or health) rather than outputs (e.g. time in school, test scores). There are a limited number of experimental studies providing direct evidence that education interventions improve the outcomes that we consider most important, such as earnings, health, and rates of marriage and fertility among teenage girls. Two recent randomized controlled trials (RCTs) estimate positive effects of secondary school scholarships on later life earnings, and we expect this evidence to improve in the future. Four RCTs provide evidence that education interventions can reduce fertility and marriage rates of young women and girls. However, there is very little evidence of effects of education on health outcomes. How cost-effective are these programs? We estimate the cost-effectiveness of interventions evaluated in three experimental studies. Our current best estimates indicate that secondary school scholarships in Ghana were in the range of cost-effectiveness of some of our other priority programs but that secondary school scholarships in Colombia were substantially less cost-effective. We think that the majority of the value of the programs in Ghana and Colombia was the result of sustained increases in earnings, rather than reductions in fertility and marriage of young women. We estimate that a small school uniform subsidy in Kenya is roughly as cost-effective as some of our other priority programs because of its negative effect on rates of teenage pregnancy and marriage, though we are very uncertain about this estimate. Areas of uncertainty: We are cautious about drawing general conclusions from individual education evaluations because we think that education interventions, and the mechanisms that they work through, are particularly complex, and there are large differences between different settings where they are implemented. The evidence that education increases earnings is currently thin, and we hope that there will be more experimental studies evaluating this relationship in the future. We are also uncertain about how much value to place on social effects of education, such as reductions in fertility and marriage of young women: our cost-effectiveness analysis relies on highly subjective value judgments which we may change our mind on in the future. Bottom line: The evidence that education interventions have effects on outcomes that we think are important has very recently improved, and we expect it to improve further in the future. We think that the existing evidence for positive effects on earnings is too thin to draw general conclusions. Although the evidence that education interventions can reduce rates of fertility and marriage among young women is stronger, we are uncertain about how to value these outcomes relative to our other priority programs.  Published: April 2018 (2009 version)

  Table of Contents    Focus of this report  Interventions considered Our approach to assessing evidence   How education may help people in low-income countries Evidence of effectiveness  Effects on labor market and pecuniary outcomes  Secondary school scholarships in Ghana (Duflo, Dupas, and Kremer 2017) Vouchers for private secondary schools in Colombia (Bettinger et al. 2014/Bettinger et al. 2017) Quasi-experimental evidence Vocational training programs   Effects on health outcomes Effects on social outcomes Effects on outputs: time in school and test scores How generalizable is this evidence?   Cost-effectiveness Areas of uncertainty Focus of further investigation Sources    Focus of this report Interventions considered There are a wide variety of programs and interventions that focus on improving education in developing countries. This report focuses on programs that aim to improve primary and secondary education in various ways. We also briefly discuss the evidence for vocational training programs for young adults above secondary school age, though this is not the main focus of this report. 

 The scope of this report does not cover programs that focus on early childhood (pre-primary) development, cash transfers, or health programs that may have impacts on education outcomes because we analyze these programs in other intervention reports.1 

 To map out the broad range of primary and secondary education programs and interventions in the scope of this report, we use the classification set out in Glewwe and Muralidharan 2016 and give examples of each type of program and intervention:

 Demand-side interventions:2 information-based interventions (e.g. providing information on returns to education, counseling services); scholarship programs of various designs (e.g. vouchers, merit-based scholarships with individual or group performance incentives); and other household interventions (e.g. parental education programs, female sanitary products, transport subsidies, school uniform subsidies). School inputs: increasing access to schools (e.g. school building programs, increased hours of schooling); providing pedagogical materials and facilities (e.g. textbooks, flipcharts, multi-level teaching materials, school facilities); increasing the quantity and quality of teachers; providing food (e.g. school meals, take-home rations); and providing resources at a large scale (e.g. block grants). Pedagogy: teaching at the right level (e.g. supplementary teaching, remedial instruction); tracking or streaming into classes based on initial learning levels or ability; using computers, electronic games, and other technology-enhanced instruction; and reading intensive pedagogy. Governance: monitoring teacher attendance to reduce absence; modifying the school's management system (e.g. decentralized 'school-based management'); paying teachers based on performance (e.g. group or individual performance pay, bonuses); modifying the structure of teacher contracts (e.g. use of contract teachers); and establishing other management practices (e.g. private-public partnerships, single-sex schools). Our approach to assessing evidence On this page, we focus on the questions we believe are (a) most likely to lead us to include an education intervention as a priority program, and (b) most tractable. In particular, we focus on:

 Evidence from randomized controlled trials (RCTs). As a first step in our review of the evidence base for educational interventions, this page focuses on experimental evidence from RCTs. We discuss quasi-experimental studies with plausible identification strategies (such as regression discontinuity and difference-in-difference designs) in our supplementary information. We hope to spend more time reviewing quasi-experimental studies in the future. Evidence of effect on outcomes (such as income, health, or social outcomes) rather than outputs (such as increased time in school or improved test scores). There are a number of variables that can be used to measure the effects of education interventions, and we place significantly more emphasis on the effects on some variables than others. We distinguish between 'outputs' of education interventions, namely whether they increase time in school or student learning (measured by test scores),3 and the effect of education interventions on people's life 'outcomes', including employment, earnings, health outcomes, fertility, and marriage. We do not place much intrinsic value on increasing time in school or test scores, although we think that such improvements may have instrumental value.4 The majority of experimental studies of the effects of education interventions focus on the effects on time in school and test scores. However, we place far more emphasis on a few recent studies that estimate the effects of education programs on life outcomes, such as earnings and rates of fertility and marriage among young women and girls. How education may help people in low-income countries There are a number of mechanisms through which education may have positive effects on individuals in low-income countries and on the general growth and development of low-income countries.5 Education may raise the long-run productivity of workers, thereby increasing individuals' wages and countries' growth and having a range of other knock-on effects on general well-being.6 Education may also improve health by improving knowledge of and access to relevant information. Education may be particularly impactful for girls, due to its potential effects on fertility and marriage decisions.7 Increased schooling of girls may also improve female empowerment. More diffuse effects of education are possible: it may improve civic participation and governance8 or reduce crime and conflict.9 Finally, it is sometimes argued that education has intrinsic value (i.e. education is itself valuable to individuals, regardless of whether it has effects on any other outcomes mentioned).10

 Evidence of effectiveness We conclude:

 There is limited evidence on the effects of education on pecuniary outcomes, though two recent working papers suggest there are significant positive effects. Two RCTs find suggestive evidence that vocational training programs can have positive effects on earnings and employment, although this report focuses on primary and secondary education rather than vocational training. The evidence of a positive effect on health outcomes is very limited. One RCT finds secondary school scholarships decreased risky sexual behavior, but another finds no effect on the rate of sexually transmitted diseases.  RCTs of four different programs find that education interventions can reduce rates of pregnancy and marriage among teenage girls and young women, though the mechanisms that drive these effects are not clear. Overall, we think that the existing evidence for positive effects on earnings is too thin to draw general conclusions. Although the evidence that education interventions can reduce rates of fertility and marriage among young women is stronger, we are uncertain about how to value these outcomes relative to our other priority programs.

 We are particularly cautious about drawing general conclusions from the evidence because education interventions and the mechanisms through which they work are often very complex and because they have been conducted in very different contexts.

 Finally, we provide a summary from a systematic review of the effect of educational interventions on outputs (time in school and test scores). We do not currently believe this evidence is sufficient to recommend any particular educational intervention as a priority program.

 Effects on labor market and pecuniary outcomes There is limited experimental evidence on the effects of primary and secondary education on labor market and pecuniary outcomes, though recent and ongoing research is improving this evidence base. We place most emphasis on the results of two recent working papers which use randomized designs to estimate the effects of a secondary school scholarship program in Ghana and a secondary school voucher program in Colombia on long-term labor market outcomes. We are not aware of any randomized controlled trials that estimate the effects of primary school interventions on labor market or pecuniary outcomes.

 We summarize these papers in the following table and discuss them in more detail below.

 Program Context Study/studies Evaluation design Main results Secondary school scholarships Students (average age of 17) admitted to senior high school in Ghana, 2008 Duflo, Dupas, and Kremer 2017 RCT, with follow-up data 8 years after intervention Vocational schools: scholarship winners more likely to work and have 24% higher earnings. Academic schools: scholarship winners more likely to be in school at time of study; too early to estimate labor market impacts. Secondary school voucher program Students aged 12-13 admitted to secondary school in Colombia, 1998 Bettinger et al. 2014/Bettinger et al. 2017 RCT, with follow-up data 8-14 years after secondary school completion Vocational schools: voucher winners have 17% higher average formal sector earnings. Academic schools: no effect on formal sector earnings.  Both papers combine randomized allocation of a secondary school scholarship or voucher with long-run follow-up data on labor market outcomes. The evidence of positive long-term effects on employment and earnings is strongest for individuals who applied to vocational secondary schools, which may better prepare students for employment. 

 We interpret these results with caution because they are still preliminary and because we think that concerns about external validity are particularly acute for education interventions, as we discuss in more detail below. We place slightly more emphasis on the results of Duflo, Dupas, and Kremer 2017 because we think we have a better understanding of the mechanisms that drive the results and because it is carried out in a setting with lower levels of education and development. 

 Details follow.

 Secondary school scholarships in Ghana (Duflo, Dupas, and Kremer 2017) The strongest evidence of the effects of an education intervention on long-term employment and income comes from a randomized controlled trial led by Esther Duflo, Pascaline Dupas, and Michael Kremer that tests the effects of secondary school scholarships in Ghana. We focus our discussion on Duflo, Dupas, and Kremer 2017, a working paper released in March 2017 which finds evidence that scholarship winners were more likely to work and have higher earnings.

 Evaluation design:

 Primary schools in Ghana do not charge fees, and enrollment is near-universal. However, enrollment is significantly lower in secondary schools, which charge large fees.11

 Duflo, Dupas, and Kremer 2017 selected a stratified sample of 2,064 individuals, who had gained admission to secondary school but had not enrolled by the end of the first term of the 2008-2009 school year, and randomly allocated a third of the children in the sample to receive a full scholarship for tuition and fees for four years.12 The scholarship was paid directly to the school and amounted to about 480 Ghana cedis per student per year (in 2016 Ghana cedis; equivalent to about $121 in 2016 US dollars).13 

 Long-term follow-up data on employment, income, and other outcomes of students in the treatment and control groups is being collected with cell phones.14 

 Results:

 Receiving a scholarship increased the probability that students completed senior high school (SHS): by 2016, 74% of scholarship winners had completed SHS, compared to only 47% of non-winners.15 The authors estimate two types of effects that we are interested in: the effects of being assigned to receive a scholarship on income, and the effects of going to school for an additional year on income.16 We discuss their estimates of the effects on income and labor market outcomes here and their estimates of non-pecuniary effects below. 

 The authors find strong evidence of positive effects of the scholarships and additional years of schooling on income and labor market outcomes in this context. They estimate that scholarship winners worked 10 hours more on average and were more likely to have positive income in the last month, despite also being more likely to be enrolled in tertiary education.17 

 The fact that scholarship winners were more likely to be enrolled in tertiary education distorts the estimates of the labor market effects of the scholarship and education, so the authors consider the effects on students who were admitted into a vocational track separately. These students were no more likely than students who did not win scholarships to be enrolled in tertiary education when data was collected in 2016.18 The authors estimate that scholarship winners admitted to vocational majors had 24% higher earnings than non-winners in the last month, though this estimate is only significant at the 10% level.19 This increase in earnings is entirely accounted for by the fact that this group of scholarship winners was more likely to be doing any work, rather than getting paid more per hour of labor or working more hours than non-winners who also work.20 Further, these increases in employment are concentrated in particular types of employment: scholarship winners were 8.5 percentage points more likely to work for a wage, and male winners were more likely to work as a day or seasonal laborer.21

 Our interpretation:

 We place significant emphasis on the results of Duflo, Dupas, and Kremer 2017 because it is a well-designed randomized controlled trial with high quality long-term data on outcomes.22 Although we believe it provides the best available evidence on the long-term effects of an education intervention on labor market outcomes, we interpret its results with caution for a number of reasons:

  This is a working paper, and some of the results are preliminary. In particular, we focus our discussion on the labor market effects of students admitted to vocational tracks in senior high schools because students admitted to academic tracks were more likely to still be in education if they received the scholarship. The authors note that it is too early to draw strong conclusions on the labor market impacts for students admitted to academic tracks.23 We do not think that these estimates are very generalizable. We are often concerned about the external validity of individual studies, and we think that this problem is particularly acute for evaluations of education interventions, as we discuss in more detail below. This evaluation was conducted in an unusually challenging macroeconomic environment and labor market in Ghana due to poor recent macroeconomic performance, changes to government hiring policies, and changes to SHS enrollment requirements that resulted in two cohorts graduating at the same time in 2012.24 In addition, the conclusions we take from this study when evaluating other interventions will depend on what these other interventions consist of. For example, if we were evaluating universal free secondary education in Ghana, we would expect the average effects to be significantly smaller, because the students selected in this study were marginal students who had been admitted to secondary school, but could not afford to go.25 We would be more confident that a scholarship program targeted at students in financial need would have effects similar to those estimated in Duflo, Dupas, and Kremer 2017. The scholarships may have had effects through non-educational channels. In particular, receipt of a scholarship represented an income transfer for families that would have sent their children to school irrespective of the scholarship. The authors discuss this in detail in section 4.2 of the paper, where they argue that the non-educational channels of scholarship effects were likely to be small.26 We think that their approach is very reasonable and note that it would be infeasible to design and implement a study that could fully rule out non-educational impacts of an education policy. The paper does not account for potential spillover effects of the scholarships. These spillover effects (or externalities) may have been positive, and the paper might therefore underestimate the benefits of the program, both for those who won and did not win the scholarships. For example, it is possible that individuals who were not assigned to receive the scholarship still benefited from the program, perhaps through peer learning effects or additional job creation by the scholarship winners. However, spillover effects may have also been negative, and the paper might therefore overestimate the benefits of the program. For example, if scholarship winners had more positive labor market outcomes at the expense of non-winners, then estimates of the direct benefit would be biased upwards, and there would be additional negative effects of the policy overall.27 The paper estimates the partial equilibrium effects of the scholarships, not the general equilibrium effects. It is possible that the scholarship program may have effects on other markets and prices that are not accounted for in the paper, especially if the policy was scaled up. The authors acknowledge this point,28 and we do not think that estimation of the full general equilibrium effects of an education policy is feasible in general. Vouchers for private secondary schools in Colombia (Bettinger et al. 2014/Bettinger et al. 2017) In this section, we discuss the results of a working paper by Eric Bettinger, Michael Kremer, Maurice Kugler, Carlos Medina, Christian Posso, and Juan E. Saavedra. The version of this paper that is publicly available is a 2014 draft, but the authors have shared a confidential internal draft of an updated version of the paper from June 2017. Although this updated draft is still preliminary and not yet publicly available, some of the results are important for our analysis. We try to make it clear which version of the paper we are referencing in what follows, and we reference the earlier version when both versions are identical.

 Apart from Duflo, Dupas, and Kremer 2017, the only other paper we are aware of that uses experimental data to evaluate the effects of a secondary education intervention on labor market outcomes is Bettinger et al. 2014/Bettinger et al. 2017, a working paper. This is a long-term follow-up evaluation of a private school voucher lottery in Colombia that estimates the effects of the program on formal sector employment and earnings using administrative data.29 The authors estimate that lottery winners had higher average formal sector earnings between 8 and 14 years after secondary school completion and find that these benefits were concentrated among males who applied to vocational secondary schools (Bettinger et al. 2017).30 We place slightly less emphasis on these estimates than those in Duflo, Dupas, and Kremer 2017 because we are more uncertain about the mechanisms driving the results. In particular, we think that it is possible that impacts are driven by income effects of the policy, though there is suggestive evidence that educational effects explain at least a portion of the estimated impacts.

 Evaluation design:

 Bettinger et al. 2014/Bettinger et al. 2017 evaluate the PACES scholarship program, which was introduced in Colombia in 1992 to improve secondary school enrollment rates of disadvantaged students by providing them with vouchers to attend private schools.31 Although the vouchers initially covered the entire tuition fees of most participating private schools, their value did not increase with inflation. By 1998, the scholarship only covered about 56% of the tuition of the average participating school.32 

 Students were aged 12-13 at the time of the application and had to apply and be accepted to a participating school to be eligible for a voucher. However, in areas where demand for vouchers exceeded the number of available private school places, vouchers were allocated by lottery.33 The PACES program has been evaluated in previous papers (Angrist et al. 2002, Angrist, Bettinger, and Kremer 2006, Bettinger, Kremer, and Saavedra 2010), though none of these looked at longer-term labor market or pecuniary outcomes.

 Results:

 The authors match data on randomized allocation of vouchers with a government dataset to estimate the effect of the policy on formal sector employment and earnings.34 Individuals only showed up in this dataset if they had worked in the formal sector between 2008 and 2014, and 80.9% of lottery winners and 80.1% of lottery losers showed up in this data, though this difference is not statistically significant.35 In addition, there is no difference in average number of months worked in the formal sector between lottery winners and losers.36 However, scholarship winners earned an additional $196 in formal annual earnings, an 8% increase, and this difference is statistically significant at the 10% level.37 

 This increase in earnings is entirely driven by individuals who applied to vocational rather than academic schools (before the scholarship lottery was held). Lottery winners who applied to vocational schools earned an additional $427 in formal annual earnings compared to lottery losers who applied to vocational schools (17% increase, statistically significant at the 5% level).38 The authors note that these effects are particularly large for men and are likely an underestimate given that 6% of vocational school scholarship winners were in tertiary education, compared to 3.9% of vocational school non-winners, in the period under study.39 They also estimate that scholarship winners paid more taxes but do not find evidence that they were any less likely to qualify for targeted benefits programs.40

 The mechanisms that explain why voucher lottery winners had higher formal sector earnings than lottery losers are very important for the broader conclusions that we draw from this paper, in particular when assessing how similar education interventions might compare to our other priority programs. However, a number of different mechanisms are possible, and we are uncertain about which ones are most likely to be driving these results.41 In what follows, we discuss each potential mechanism and how likely we think it is to be driving the effects on formal sector earnings: 

 More time in school. Although voucher winners were significantly more likely to complete secondary school on time, they experienced small changes in the number of years of secondary school completed, so increased secondary school attainment is unlikely to explain the effects on formal sector earnings. The effects on secondary school attainment are similar for vocational and academic applicants.42 However, vocational school applicants that won the voucher lottery scored higher in the college admission test than lottery losers who applied to vocational schools, were more likely to enroll in tertiary education (25.8% vs. 18.8%), and attained more years of tertiary education on average (0.61 vs. 0.42).43 Academic applicants who won the voucher lottery were no more likely to enroll in tertiary education and did not complete more years of tertiary education.44 In addition, the earnings benefits for vocational applicants who won the lottery are concentrated among higher earners, who we assume were more likely to have tertiary education.45 It therefore seems likely that this increase in tertiary education for vocational applicants who won the lottery explains a significant proportion of the increase in formal sector earnings. Higher quality of schooling received. There are a few ways in which winning the voucher lottery may allow children to attend higher quality schools: by allowing children to attend (more expensive) private schools, which might be higher quality; by increasing choice of schools, allowing a child to go to a school that is a better personal match; by improving the 'quality' of a child's classmates or peers; by increasing choice and competition, which might raise school standards; or by decreasing congestion in schools. Of these channels, we think the first two are more likely to explain the increase in formal sector earnings for voucher winners, though we do not have evidence that directly supports this hypothesis. The fourth and fifth channels would likely raise the quality of schooling for non-winners as well as winners, and the authors give suggestive evidence that scholarship winners' peers are less desirable on average, so the final three channels are not likely to explain the effects of the scholarship.46 

 Incentives to avoid grade repetition. The policy had elements of a merit scholarship program: renewal of the scholarship was conditional on students successfully completing their grade. However, the effects of the program are strongest among those at the top of the distribution, suggesting that this mechanism was not driving the effects of the program.47 Decreased movement between schools. The authors note that it was administratively difficult to retain the scholarship for individuals who transferred schools and that scholarship winners were therefore less likely to change schools.48 It is possible that this benefitted scholarship winners, and it might explain part of the effects of the program. Income effects. For individuals who would have attended a private school anyway, winning the voucher lottery represented an income transfer to their family, which might have led to non-educational effects of the program. These individuals were numerous: 88% of lottery losers attended private school in the first year of the program, 54% in the third year, and 32% in the sixth year.49 These income effects may therefore be important in explaining the long-term effects of the program. However, the proportion of lottery losers who attended private schools was essentially the same for vocational and academic school applicants, so this mechanism does not explain the difference in effects of the scholarship for these groups.50 Earlier entry to the workforce. If formal sector earnings depend on the length of time a person has spent in the workforce, and if scholarship winners were more likely to enter the workforce earlier, this might explain some of the observed effects of the policy. Given that scholarship winners graduated from secondary school earlier on average, it seems likely that they might have entered the workforce earlier.51 In addition, there is suggestive evidence that scholarship winners that applied to vocational schools completed tertiary education sooner than those who applied to academic schools, though this evidence is too weak to draw any conclusions.52 Earlier entry to the workforce might have been the result of income effects of the policy (e.g. by allowing individuals to enroll in tertiary education sooner and complete it without interruptions) or educational effects of the policy (e.g. by reducing school year repetitions). It seems likely that at least some of the effects of the policy on formal sector earnings were educational: although the income transfers generated by the policy were large, they do not appear to explain the fact that scholarship winners who applied to vocational schools had increased formal sector earnings, but those who applied to academic schools did not. The authors suggest that vocational schools are more responsive to labor markets and better prepare students for tertiary education and the labor market, and they find some evidence for this hypothesis.53 This raises the interesting possibility that parents might have been misinformed about the relative returns to academic and vocational secondary schooling and that the policy was effective because it kept children from switching from vocational to academic schools, despite academic schools being more prestigious.54

 Our interpretation:

 We place less emphasis on the results of this paper than we do on the results of Duflo, Dupas, and Kremer 2017, mainly because we are more uncertain about the mechanisms that are driving the observed increase in formal sector earnings among scholarship winners who applied to vocational schools. Whereas provision of secondary school scholarships in Ghana studied by Duflo, Dupas, and Kremer 2017 was a relatively straightforward program that increased secondary school attainment of scholarship winners, there are many more potential mechanisms that might have contributed to the effects of the Colombian vouchers program evaluated in Bettinger et al. 2014/Bettinger et al. 2017. 

 As in our discussion of Duflo, Dupas, and Kremer 2017, we also note that Bettinger et al. 2017 is still a working paper with some preliminary analysis and that it cannot estimate potential spillovers of the policy (e.g. perhaps voucher winners get higher paying formal sector jobs at the expense of scholarship losers). 

 Finally, we note that our discussion has centered on the effect of the policy on formal sector earnings because informal sector earnings are not included in the main administrative dataset used by the authors.55 However, the authors present evidence which indicates that the increase in formal sector earnings was not offset by a corresponding decrease in informal sector earnings.56

 Quasi-experimental evidence There are a number of studies that use quasi-experimental methods to estimate the impact of secondary and primary education interventions on labor market outcomes. We place less emphasis on their results relative to Duflo, Dupas, and Kremer 2017 and Bettinger et al. 2017 and have not analyzed them in more detail because we think quasi-experimental methods are more susceptible to bias than randomized trials, as we have previously written about.57

 In general, we think these studies provide weak supporting evidence to Duflo, Dupas, and Kremer 2017 and Bettinger et al. 2017 and indicate that education interventions may have positive effects on labor market outcomes, particularly for women (Keats 201658, Chicoine 2017, Brudevold-Newman 2016) and possibly with negative consequences for individuals that did not benefit from the education intervention (Duflo 2004).

 We give an overview of the quasi-experimental studies we have found in our supplementary information.

 Vocational training programs This report focuses on interventions in primary and secondary education, as noted above. However, in this section we give a brief overview of research that estimates the effects of 'vocational training' programs on pecuniary and labor market outcomes. 

 Vocational training programs vary significantly in their content and design, but most consist of non-academic training for young adults, often focusing on those who are unemployed or lacking skills or education. Programs may vary in a number of ways, including the type of institutions providing training (e.g. private sector, public sector, NGOs, and church groups), the size of the program, the skills being taught (e.g. entrepreneurship, skilled construction such as carpentry, or automotive mechanics), and the length, structure, and content of the program.59

 Overall, there is evidence from RCTs of two vocational training programs, one conducted in Colombia and the other in the Dominican Republic, that demonstrate positive effects on earnings and (formal sector) employment. However, as we discuss in more detail below, we think that it is difficult to draw general conclusions from individual evaluations of education interventions, and these challenges are particularly acute for vocational training programs, given the wide variety of programs available within and between countries. 

 We know of a few papers that evaluate randomized trials and a number of ongoing randomized controlled trials that evaluate the effects of vocational training programs. The following table gives a summary of these papers, and we discuss them in more detail below.

 Program Context Study/studies Evaluation design Main results Vocational training program Unemployed youths aged 18-25 in Colombia, 2005 Attanasio, Kugler, and Meghir 2011Attanasio et al. 2017Kugler et al. 2015 RCT, with long-run follow-up data 13-15 months after program end: increased earnings and employment for women; no effects on men. 3-9 years after program end: 17.5% higher formal earnings for women; 10.7% higher formal earnings for men.  Job training program Low-income youths aged 18-29 in the Dominican Republic, 2001-2006  Card et al. 2011Ibarrarán et al. 2016 RCT, with long-run follow-up data 10-14 months after program end: no effect on employment outcomes. 6 years after program introduction: positive effect on earnings in the capital city. Technical and Vocational Vouchers Program Out-of-school youth in Kenya, 2008 Hicks et al. 201360 RCT, with long-run follow-up data Research ongoing; results not yet available. Apprenticeship training in Ghana Young people with limited education in Ghana, 2012 Hardy, Mbiti, and McCasland 201661  RCT, with long-run follow-up data Research ongoing; results not yet available. Completed RCTs

 Three papers use a randomized trial to evaluate a vocational training program for unemployed youths (aged 18 to 25) in Colombia that was introduced in 2005.62 Attanasio, Kugler, and Meghir 2011 estimates the impact of the training program 13-15 months after its implementation and finds strong evidence that it increased earnings and employment for women, though there were no statistically significant effects for men, apart from a shift to working in the formal sector.63 Attanasio et al. 2017 uses administrative data to estimate the impacts of the program between 3 and 9 years after it ended.64 It finds that program effects persisted, increasing formal sector earnings and participation, with effects similar for men and women.65 Kugler et al. 2015 shows that vocational training and formal education were complementary investments in this setting, with participants more likely to complete secondary education and enroll in tertiary education.66 Card et al. 2011 and Ibarrarán et al. 2016 use a randomized trial to evaluate the impacts of a job training program for low-income youths (ages 18-29) with less than a secondary education in the Dominican Republic.67 Card et al. 2011 used a follow-up survey 10-14 months after the end of the program and finds little evidence of a positive effect on employment outcomes, though it notes that there were problems with the evaluation design.68 A recent working paper, Ibarrarán et al. 2016, estimates the impacts of the program six years after its introduction and finds evidence that it increased formal sector employment but had no effect on overall employment.69 The authors also estimate that the program had a positive effect on earnings of youth in the capital city.70 Ongoing RCTs

 Hicks et al. 2013 is a preliminary and incomplete working paper that gives initial results of an ongoing randomized trial in Kenya. It estimates the effects of the Technical and Vocational Vouchers Program (TVVP), which began in 2008 and randomly allocated vouchers for vocational training to out-of-school youth in Kenya.71 The authors collected two waves of follow-up data.72 Preliminary results from the first wave, which was collected in 2012, provided no evidence of sectoral shifts and only limited evidence that the program increased wages among wage earners.73 The second wave of data collection ended in fall of 2016, and the authors indicated that preliminary results from this wave will soon be forthcoming. Evaluation of a one-year apprenticeship training program in Ghana is ongoing, with data collection scheduled for completion in 2017. Preliminary descriptive statistics are reported in Hardy, Mbiti, and McCasland 2016. There is also an ongoing evaluation of a youth technical vocational training in Mongolia, though we are not yet aware of any results from this study. We are keeping track of the results of a randomized trial evaluating the vocational training program carried out by Educate!. We have published conversation notes with Loren Crary, the Director of External Relations and Counsel at Educate!. Effects on health outcomes Overall, we think there is only weak evidence that education programs improve health outcomes. We are aware of only two studies that use a randomized trial to test the effects of an education intervention on health outcomes. 

 We have not read these papers in detail, but we summarize their key findings in the following table and discuss them in more detail below.

 Program Context Study/studies Evaluation design Main results Secondary school scholarships Students (average age of 17) admitted to senior high school in Ghana, 2008 Duflo, Dupas, and Kremer 2017 RCT, with follow-up data 8 years after intervention Males: less risky sexual behavior; less exposure to STIs; more preventative health behaviors; no effect on mental health or life satisfaction. Females: more preventative health behaviors; no effect on sexual behavior, exposure to STIs, mental health, or life satisfaction. Secondary school subsidy (free uniforms) Upper primary school students (average age of 13.5) in Western Kenya, 2003 Duflo, Dupas, and Kremer 2015 RCT, with follow-up data 7 years after start of intervention Subsidy alone: no reduction in STIs. Subsidy combined with abstinence education: reduction in STIs. Experimental evidence:

 Duflo, Dupas, and Kremer 2017 uses a randomized trial to test the long-run effects of secondary school scholarships on students in Ghana, as described in more detail above. Using survey data from 2013 (5 years after the start of the scholarship), the authors estimate that male scholarship winners adopted less risky sexual behavior (self-reported)74 and had less exposure to sexually transmitted infections (self-reported), with no significant change in risky sexual behavior or exposure to STIs for women. Both male and female scholarship winners that were admitted to academic tracks adopted more preventative health behaviors (more handwashing, bednet use, and mosquito repellent use), though students admitted to vocational tracks experienced no statistically significant change in preventative health behaviors.75 It is challenging to place an intuitive value on the size of these effects, as they are indexes constructed from responses to survey questions, but our interpretation is that they are small relative to the earnings gains resulting from this intervention.76 We explicitly compare the size of effects of the program on individual health outcomes to the effects on earnings in our cost-effectiveness analysis, and we calculate that the health behavior gains estimated in this context provide less than 1% of the value of the estimated earnings gains. The authors did not find any evidence of effects of scholarships on indices of mental health and life satisfaction measurements.77 Duflo, Dupas, and Kremer 2015 uses an RCT to evaluate the effects of a school subsidy program (free school uniforms), teacher training to deliver a government program promoting abstinence until marriage, and the combination of both programs on outcomes of upper primary school students in Western Kenya from 2003 to 2010.78 The authors test participants for two sexually transmitted infections, HIV and Herpes simplex virus type 2 (HSV2), over 7 years. HIV prevalence was very low in their sample, so the authors focus on the effects on HSV2.79 They do not find evidence that the education subsidy alone reduced infection of HSV2, though the combination of the subsidy and the 'abstinence until marriage' education program reduced the rate of HSV2 infection among girls over 7 years (11.8% to 9.5%).80 The mechanisms for this effect are not clear, though the authors hypothesize that the education subsidies led girls to have less unprotected sex and that the abstinence until marriage program resulted in fewer casual relationships, resulting in the joint effect of fewer STIs.81 

 Quasi-experimental evidence: We have found three additional studies that use quasi-experimental methods to estimate the impact of secondary and primary education interventions on health outcomes. We did not incorporate those studies into this page but summarize them in our supplementary information for the interested reader.82 

 Effects on social outcomes RCTs of four different programs find that education interventions can reduce rates of pregnancy and marriage among teenage girls and young women, though the mechanisms that drive these effects are not clear.83 

 We are uncertain how to value these outcomes relative to health or consumption gains, particularly when making a comparison between education programs and our priority programs. We address this explicitly in our cost-effectiveness analysis. 

 The following table summarizes the main results of the randomized controlled trials that estimate the effects of education policies on social outcomes, and we discuss these studies in more detail below.

 Program Context Study/studies Evaluation design Main results Secondary school scholarships Students (average age of 17) admitted to senior high school in Ghana, 2008 Duflo, Dupas, and Kremer 2017 RCT, with follow-up data 8 years after intervention Women, at age 25: reduced pregnancies; reduced unwanted pregnancies; fewer children; no effect on desired number of children.  Secondary school voucher program Students aged 12-13 admitted to secondary school in Colombia, 1998 Bettinger et al. 2014/Bettinger et al. 2017 RCT, with follow-up data 8-14 years after secondary school completion Girls/women: reduced teenage pregnancy; no effect on number of children at age 30. Boys: reduced proportion with partners with child. Secondary school subsidy (free uniforms) Upper primary school students (average age of 13.5) in Western Kenya, 2003 Duflo, Dupas, and Kremer 2015 RCT, with follow-up data 7 years after start of intervention Girls: reduced teenage pregnancy; reduced probability of marriage by age 21. Unconditional cash transfer (UCT) and conditional cash transfer (CCT) Adolescent girls in Zomba, Malawi, 2007. CCT conditional on school attendance. Baird et al. 2010Baird, McIntosh, and Özler 2011 RCT, with follow-up data 1 year after start of intervention CCT results in increased educational attainment, but reductions in pregnancy and marriage seem to be driven by income effects of UCT.  Experimental evidence:

 Duflo, Dupas, and Kremer 2017 uses a randomized trial to test the long-run effects of secondary school scholarships on students in Ghana, as described in more detail above. The authors estimate large and statistically significant (at the 1% level) effects on women's marriage and fertility outcomes at age 25 but do not find evidence of impacts for men. At an average age of 25, female scholarship winners were less likely to have ever lived with a partner (34% to 25%), less likely to have ever been pregnant (58% to 48%), less likely to have had an unwanted first pregnancy (57% to 45%), and had had 0.217 fewer children on average. However, women experienced no statistically significant change in desired number of children at an average age of 22.84 These effects are also statistically significant in earlier rounds of data collection (at average ages of 22 and 24).85 Reductions in the rate of pregnancy persisted after women left school, suggesting that this was not just an 'incarceration effect' but more likely the result of an increased opportunity cost of having children, access to improved information, and/or changes to preferences.86 Bettinger et al. 2014/Bettinger et al. 2017 uses a randomized trial to estimate the long-run effects of secondary school vouchers in Colombia, as described in more detail above. The authors estimate large effects of the scholarship on the proportion of girls who had a child in their teenage years (23.4% to 19.1%) and on the proportion of boys who had a partner who had a child as a teenager (16.1% to 11.0%).87 In contrast to the effects on formal sector earnings, the effects on fertility are concentrated among applicants to academic schools.88 However, the authors do not find evidence of a reduction in the total number of children scholarship lottery winners had at age 30, suggesting that the reduction in teen fertility is a result of delayed fertility rather than a permanent increase in the opportunity cost of having a child.89 Duflo, Dupas, and Kremer 2015 uses an RCT to evaluate the effects of a school subsidy program (free school uniforms), teacher training to deliver a government program promoting abstinence until marriage, and the combination of both programs on outcomes of upper school primary students in Western Kenya from 2003 to 2010, as noted above. The authors estimate that the education subsidies (implemented alone) reduced the teenage pregnancy rate for girls (16% to 13%) and that this difference in fertility persisted 7 years after the intervention (49% versus 46%), although the difference is no longer statistically significant at this time.90 The education subsidies also reduced girls' probability of getting married by age 21 (39% to 35%).91 Two papers estimate the effects of a conditional cash transfer (CCT) program in Zomba, Malawi. Baird et al. 2010 uses a randomized trial to evaluate the Zomba Cash Transfer Program, which offers cash transfers and payment of school fees to young women who stay in or return to school.92 After one year, the program had large positive effects on school enrollment for both girls in school at baseline (89.1% versus 93%) and girls not in school at baseline (17.2% versus 61.4%). For girls who were not in school at baseline, the program (after one year) had large negative effects on the probability of early marriage (27.7% to 16.4%), the probability of teenage pregnancy (16.2% to 11.1%), and self-reported sexual activity. Effects on marriage, pregnancy, and sexual activity for girls who were in school at the start of the intervention were either much smaller or statistically insignificant.93 It is unclear to what extent these effects were caused by increased education rather than income effects of the policy. However, Baird, McIntosh, and Özler 2011 helps us to understand which mechanism is more important in explaining these effects in this context by comparing an unconditional cash transfer program (UCT) to a CCT.94 Although the CCT had a larger effect on educational outcomes, it did not have a significant effect on marriage or pregnancy, whereas the UCT reduced both, suggesting that increases in income are principally driving these effects.95 Quasi-experimental evidence:

 There are a number of studies that use quasi-experimental methods to estimate the impact of secondary and primary education interventions on rates of fertility and marriage among females. We did not incorporate those studies into this page but summarize them on another page for the interested reader. We interpret them as evidence that is broadly supportive of the conclusions of the randomized controlled trials above, that education interventions can reduce rates of fertility and marriage.

 Mechanisms:

 There is mixed evidence on whether reductions in teenage pregnancy are driven primarily by 'incarceration' effects of schooling, whereby girls choose to delay pregnancy while in school, or persistent changes to preferences, information, and/or the 'opportunity cost' of children, which imply that fertility rates would remain lower in the long term.96 We believe that the relative importance of these two mechanisms likely depends on the specific context of an intervention. 

 We are also uncertain whether the ‘opportunity cost’ effects on pregnancy and marriage are driven primarily by educational effects or income effects of the interventions evaluated in these studies. This has important implications for our interpretation of this evidence and the conclusions we draw from it.97 Each of the four studies that use randomized controlled trials evaluate the impacts of interventions which may have income effects as well as educational effects, but the extent to which these income effects are likely to be important varies:98 

 The school uniforms subsidy evaluated in Duflo, Dupas, and Kremer 2015 is small ($12), and income effects are therefore unlikely to fully explain the estimated effects of the policy. Duflo, Dupas, and Kremer 2017 evaluates a much more generous program but makes a reasonably convincing argument that the overall income effects of the scholarship program it evaluates are likely very small (as discussed in more detail above).99 However, evidence from Baird et al. 2010 suggests that effects on marriage and fertility are primarily driven by income effects of cash transfer programs, implying that (targeted) unconditional cash transfer programs may be more cost-effective at reducing teenage pregnancy and marriage than education programs. Overall, we are uncertain to what extent income and education effects of these policies are driving these results and think this likely depends on the specific context of a policy. Effects on outputs: time in school and test scores Our current understanding is that there is limited high quality evidence establishing a causal link between increased time in school or test scores to improvements in life outcomes such as earnings or health, which we consider to be better measurements of general wellbeing.100

 We therefore place little emphasis on the effect of education interventions on the outputs discussed in this section (measurements of time in school and test scores), relative to the effects on outcomes (labor market, pecuniary, health, and social outcomes).

 We include the summary results of the most recent comprehensive systematic review we found, Glewwe and Muralidharan 2016, below.101 Due to our focus on outcomes rather than outputs, we have not independently assessed the individual studies in this review or used it to inform our conclusions. We include the summary results of the review in the table below for the interested reader and to inform possible future research if we later conclude that educational outputs are sufficient to inform our recommendations.

 Category Intervention Conclusion of Glewwe and Muralidharan 2016 Demand-side interventions Information-based interventions (for example providing information to children or parents about the returns to education). Two RCTs found suggestive evidence of increased time spent in school but no evidence of increased test scores.102  Scholarship programs Two RCTs of the same program found evidence of increased time spent in school.103 Four RCTs found evidence of improved test scores.104  Providing information to mothers about how to develop their child's learning One RCT found a small increase in test scores but no evidence of increased time in school. 105 School inputs Increasing access to schools (e.g. school building programs) Two RCTs and three quasi-experimental studies found school building programs increased time in school. Two of these studies also found evidence of increased test scores. Two quasi-experimental studies found suggestive evidence that longer school days increased test scores.106  Provision of pedagogical materials Two RCTs found no evidence that provision of textbooks increased time in school or test scores. One RCT found no evidence that flip charts increased test scores. One RCT found no evidence that libraries increased test scores. One RCT found suggestive evidence that learning materials designed for children at different learning levels increased test scores.107  Increasing the teacher-to-pupil ratio Two quasi-experimental studies found that decreasing the teacher-to-pupil ratio decreased test scores, but one RCT found no statistically significant effect. No studies could isolate the effect of extra teachers on time in school.108  Provision of food Six studies found school meals generally did not increase time in school. One study found take-home rations improved test scores. 109  Investing in school infrastructure One RCT found no effect of infrastructure improvements on time in school. One quasi-experimental study found some evidence of an increase in test scores, although it is difficult to separate infrastructure improvements from other components of the intervention.110 Pedagogy interventions Supplemental remedial instruction and 'Teaching at the right level' Three studies found interventions that focus on 'teaching at the right level' generally increased test scores. One of these studies found no effect on time in school.111  Tracking or streaming programs One RCT found tracking pupils increased test scores.112   Computers or electronic games One quasi-experimental study found increased access to computers did not increase time in school. Nine studies found that computers or electronic games can have positive or negative effects on test scores. There is large variance in the estimated size of impact, suggesting that the specific context and type of intervention is very important.113 Governance interventions Teacher monitoring programs Four RCTs study the effect of teacher monitoring on test scores; one found a significant effect, and three found no significant effect.114   School-based management interventions (decentralizing management to schools). Three RCTs and two quasi-experimental studies generally found no significant effect on time in school. Five RCTs and two quasi-experimental studies found mixed results on test scores.115   Teacher performance pay Four studies generally found positive effects on test scores. There is no evidence of increases in time in school.116  Hiring contract teachers Two RCTs found hiring contract teachers increased test scores.117  How generalizable is this evidence? We are cautious about drawing general conclusions based on evidence from individual studies carried out in specific contexts, as we have previously written about. We think that it is particularly challenging to draw general conclusions about the likely effects of education interventions for a number of reasons:

 Education interventions are often complex, and there is a high degree of heterogeneity between them. Even when trying to draw conclusions about a single type of education program with multiple experimental studies, there are often many differences between the interventions in each study. For example, scholarship programs can vary in a number of ways, including: the size of the scholarship, the (type of) schools the scholarship covers, the timing of the payment, whether there is an incentive structure, and how incentives are structured.118 The mechanisms through which education interventions work are often complex and not well understood. The effects of an education intervention will depend on an individual's other educational investments, including other interventions in the past, present, and future.119 In addition, a given intervention might work through various channels that cannot be distinguished by a researcher but may have implications for the conclusions we draw from its estimated effects. For example, we are uncertain whether the effects of education interventions on marriage and fertility rates of girls are driven primarily by income effects or educational effects, as discussed above, and this has an impact on the conclusions we draw from this evidence. The context in which education interventions are carried out can vary significantly. Empirical studies conducted at a particular point in time may yield estimated effects that are significantly different to the effects in other time periods. For example, the estimated effects of school building programs in the 1960s and 1970s (evaluated in Duflo 2001, Duflo 2004, Breierova and Duflo 2004, Chou et al. 2010) may be very different to the effects of a similar program today. Similarly, Duflo, Dupas, and Kremer 2017 discusses how particular labor market and macroeconomic conditions at the time of their study may affect its results.120 Many other aspects relating to the context in which a study is carried out may also affect its results, including where a study was conducted, which population was studied, the age of participants, and the opportunities available outside of education. There is significant heterogeneity in how outcome variables are measured in different studies. This is perhaps most significant in studies that use test scores, as these can vary in many ways, including the type of test used (standardized tests, subject tests, reading tests, numeracy tests, and those that try to measure cognitive ability) and how the tests are assessed (e.g. absolute or relative performance). There are also challenges in consistently measuring outcomes across studies. For example, earnings can be measured using self-reported data or administrative data and can include formal and/or informal income as well as income from non-labor sources. In addition, it can be difficult to know how survey questions were framed by enumerators and interpreted by respondents and therefore how to interpret them as a researcher (e.g. what counts as an 'unwanted pregnancy' in Duflo, Dupas, and Kremer 2017).121 The optimal policy or intervention is likely to be different for different age groups and for different individual students. There is significant heterogeneity in the effects of interventions at different stages of education and for different individual students, which adds to the uncertainty about whether the effects of an intervention in one context are likely to generalize to another context.122 We think that the evidence for an effect of education interventions on labor market outcomes (including earnings) is currently too limited to draw general conclusions with much confidence. However, the evidence has very recently improved, most notably as a result of two recent working papers, Duflo, Dupas, and Kremer 2017 and Bettinger et al. 2017, and we hope that this encourages more high quality research on the long-run impacts of education on labor market outcomes. 

 The evidence that education interventions reduce the marriage and fertility rate of young women and girls is stronger, but there is still significant uncertainty about the size of these effects and the mechanisms that drive them.123 As discussed above, we think that effects are likely to vary significantly in different contexts and with different interventions, and we are therefore cautious about making general recommendations of education programs based on this evidence.

 Cost-effectiveness We constructed preliminary cost-effectiveness analyses for three education interventions for which we believe there is reasonably strong evidence of impact (although we believe it would be difficult to generalize from this evidence to a program being carried out by a charity). 

 In our experience, when we eventually identify and recommend a charity implementing a program, the cost per person reached is often significantly higher than it appeared in our preliminary estimates.

 Note that our cost-effectiveness analyses are simplified models that do not take into account a number of factors. There are limitations to this kind of cost-effectiveness analysis, and we believe that cost-effectiveness estimates such as these should not be taken literally, due to the significant uncertainty around them. We provide these estimates (a) for comparative purposes and (b) because working on them helps us ensure that we are thinking through as many of the relevant issues as possible.

 Our best guess, based on these analyses, is that:

 The secondary school scholarship program in Ghana that was evaluated in Duflo, Dupas, and Kremer 2017 is in the range of cost-effectiveness of our priority programs (although towards the less favourable end).124 The secondary school vouchers program in Colombia that was evaluated in Bettinger et al. 2017 is substantially less cost-effective than our priority programs.125  The school uniform subsidy program in Kenya that was evaluated in Duflo, Dupas, and Kremer 2015 is in the range of cost-effectiveness of our priority programs (although towards the less favourable end).126 For the first two analyses, we estimate that the majority of the value of these programs came from sustained increases in earnings rather than from non-pecuniary effects, such as reductions in fertility or marriage. 

 Major uncertainties in our model include:

 Our biggest area of uncertainty is how to weight non-pecuniary outcomes relative to increases in earnings. We encourage interested readers to make a copy of our analysis and input weights reflecting their own subjective value judgements. Some of these subjective judgements are particularly difficult to make without understanding how questions were interpreted at the time of data collection. For example, it is not clear how an 'unwanted pregnancy' would be interpreted by respondents in Duflo, Dupas, and Kremer 2017. We analyze the cost-effectiveness from the point of view of a hypothetical charity implementing the specific policies in the specific contexts evaluated in the studies. We do not consider potential co-funders or the effect of the intervention on government policies and/or revenue. We may update this analysis in the future to better account for these potential effects, particularly if we evaluate a specific charity executing programs similar to those evaluated in the studies. There are a few reasonable ways to treat the costs and benefits of these particular policies. For example, in our analysis of the scholarship program in Ghana (Duflo, Dupas, and Kremer 2017), we estimate the foregone earnings of children while they are at school but do not consider the income effects of the policy to families who would have sent their child to school anyway because it is not clear how this gain would be distributed within a family.127 In our analysis of the voucher program in Colombia (Bettinger et al. 2017), we calculate foregone earnings only for the additional years of schooling, as the vast majority of children would be in school without the policy, and do not include in-kind costs of schooling or the income effects of the voucher for families who would have sent their children to private schools without the program. We also do not separately estimate the effects of the policy on children who applied to academic and vocational schools. However, we do not think that these assumptions have a significant impact on our overall cost-effectiveness estimates. Areas of uncertainty There are a number of areas of this report about which we remain uncertain. Improved clarity on the following issues would strengthen our confidence in our assessment of how promising education interventions are as a potential priority program:

 How generalizable is the evidence of effectiveness? As we discussed in more detail above, our key area of uncertainty is how well results from specific evaluations of specific policies in specific settings would generalize. We think that the problem of weak external validity of individual studies is particularly acute for education interventions. How to value social effects of education. As discussed in more detail in our cost-effectiveness analysis, we are very uncertain how to value the effects of education on social outcomes, such as reduced rates of fertility and marriage of young women and teenage girls. Our current best guesses indicate that the school uniform subsidy evaluated in Duflo, Dupas, and Kremer 2015 is roughly as cost-effective as GiveDirectly's unconditional cash transfer program, but this estimate depends on highly subjective value judgements which we could easily change our view on. How to value the other (potential) effects of education. There are a number of (potential) effects of education that we do not place much emphasis on in this report, which other people might reasonably consider in more depth in an assessment of education. These fall into three categories: We have not yet carefully reviewed the extensive literature estimating the effects of education on time in school and test scores. There is a range of other outcomes for which there is some evidence that education interventions might have an effect on, for example knowledge about politics or access to the internet. We do not place any emphasis on these outcomes in this report because we are very uncertain how to value them and think that they are likely to be significantly less important than the outcomes we do emphasize. Similarly, there are a range of outcomes which are inherently diffuse and very difficult to measure and place a value on, such as the effects of education on democracy and governance. Such effects are potentially large, but we do not know of any high quality evidence establishing a causal link between education and these outcomes and do not expect this to change. How to define an 'intervention'. When assessing the evidence for an intervention, how narrowly we define an intervention can have important consequences for the conclusions that we draw about it.128 For example, when assessing whether specific interventions reduce teenage pregnancy and marriage, if we define the intervention as 'school uniform subsidies', then the available evidence (Duflo, Dupas, and Kremer 2015) points towards significant effects. However, if we define the intervention slightly more broadly as 'small monetary incentives to attend school', then we also must consider the findings in Baird, McIntosh, and Özler 2011 and, in this context, income effects are primarily responsible for the estimated impacts, while increased educational attainment does not lead to significant effects. The evidence for positive effects on earnings. The evidence that education interventions can increase earnings is limited. Furthermore, the strongest evidence we are aware of (Duflo, Dupas, and Kremer 2017 and Bettinger et al. 2017) estimates that increases in earnings are driven by children who attend (or applied to) vocational schools, rather than academic schools, as discussed in more detail above. We expect this evidence to improve over time, particularly for children who attend academic schools, as the authors of Duflo, Dupas, and Kremer 2017 collect more rounds of data. To further strengthen this evidence, we would like to see more high quality long-term studies of the effects of a range of education interventions on labor market outcomes and earnings. Vocational training programs. Vocational training programs for young adults are not the focus of this report, as discussed above, but we plan on assessing this evidence in more detail in future work. Focus of further investigation Given the areas of uncertainty above, we expect future work to focus on:

 A more detailed review of the effects of vocational training programs. Reviewing the quasi-experimental studies we included in our supplementary information. Further investigation of the link between educational outputs (test scores and time in school) and outcomes.Sources Document Source Abdul Latif Jameel Poverty Action Lab website, "Teacher Training and Entrepreneurship Education: Evidence from a Curriculum Reform in Rwanda" Source (archive) Acemoglu et al. 2005 Source (archive) Angrist et al. 2002 Source (archive) Angrist, Bettinger, and Kremer 2006 Source (archive) Attanasio et al. 2017 Source (archive) Attanasio, Kugler, and Meghir 2011 Source (archive) Baird et al. 2010 Source (archive) Baird, McIntosh, and Özler 2011 Source (archive) Banerjee et al. 2013 Source (archive) Behrman 2010 Source (archive) Bettinger et al. 2014 Source Bettinger et al. 2017 Unpublished Bettinger, Kremer, and Saavedra 2010 Source (archive) Breierova and Duflo 2004 Source (archive) Brudevold-Newman 2016 Source (archive) Card 2001 Source (archive) Card et al. 2011 Source (archive) Chicoine 2017 Source (archive) Chou et al. 2010 Source (archive) Crépon et al. 2013 Source (archive) Dedvukaj 2016 Source (archive) Duflo 2001 Source (archive) Duflo 2004 Source (archive) Duflo, Dupas, and Kremer 2015 Source (archive) Duflo, Dupas, and Kremer 2017 Source (archive) Evans and Popova 2016 Source (archive) Evans and Yuan 2017 Source (archive) Exchange-Rates.org website, "Ghanaian Cedis (GHS) to US Dollars (USD) exchange rate for June 30, 2016" Source (archive) Ganimian and Murnane 2014 Source (archive) GiveWell's non-verbatim summary of a conversation with Loren Crary, January 10, 2017 Source GiveWell's preliminary CEA for education interventions Source Glewwe and Kremer 2006 Source (archive) Glewwe and Muralidharan 2016 Source (archive) Glewwe et al. 2012 Source (archive) Hardy, Mbiti, and McCasland 2016 Unpublished Hicks et al. 2013 Unpublished Ibarrarán et al. 2016 Source (archive) Innovations for Poverty Action website, "Returns to Apprenticeship Training in Ghana" Source (archive) Innovations for Poverty Action website, "Returns to Secondary Schooling in Ghana" Source (archive) Innovations for Poverty Action website, "Technical Vocational Training in Mongolia" Source (archive) Jensen 2012 Source (archive) Keats 2016 Unpublished Kremer, Brannen, and Glennerster 2013 Source (archive) Kugler et al. 2015 Source (archive) Lochner and Moretti 2004 Source (archive) McEwan 2015 Source (archive) Montenegro and Patrinos 2013 Source (archive) Oye, Pritchett, and Sandefur 2016 Source (archive) Pritchett 2001 Source (archive) Twitter, Justin Sandefur conversation with Pascaline Dupas 2017 Source (archive) Westacott 2017 Source (archive) Wikipedia, "Raven's Progressive Matrices" Source (archive) Filmer and Schady 2014 Source (archive) Osili and Long 2008 Source (archive) Ozier 2016 Source (archive) Wikipedia, "Difference in differences" Source (archive) Wikipedia, "Regression discontinuity design" Source (archive) Wikipedia, "Regression discontinuity design: Extensions: Fuzzy RDD" Source (archive) 1. We have not yet completed an intervention report for early childhood development interventions.

 

 2. Demand-side interventions affect the demand by households or individuals for schooling, typically by either reducing the cost of schooling or increasing its (perceived) returns.

 

 3. Time in school is measured in a variety of ways, including enrollment or school participation rates, attendance rates (e.g. number of days a child is in school), years of schooling attainment, and graduation rates. Improvements in learning are typically measured using test scores, although there are a wide variety of tests that can be used, including standardized tests, school subject tests, reading tests, numeracy tests, and tests that try to measure cognitive ability (e.g. Wikipedia, "Raven's Progressive Matrices").

 

 4. "Many see secondary education as having potentially transformative economic and social impacts, particularly for girls… This debate is surprisingly uninformed by high quality evidence from the developing world. Many studies in the developed world have used natural experiments to estimate the rates of return to education (e.g., Angrist and Krueger, 1990). However, it is not clear if the results generalize to developing countries which have vastly greater levels of education than did developed countries when they had comparable income levels (Pritchett, 2001). While many studies document the positive correlation between education and other outcomes, there are surprisingly few well-identified studies from lower income countries on the causal impacts of education. We are aware of no randomized controlled trial (RCT) and only one study based on regression discontinuities in admission test scores on the labor market impact of secondary education (Ozier, 2016). Although there are strong claims about the effects of secondary education for girls, especially on reproductive health, fertility, and empowerment (UNGEI, 2010; Warner, Malhotra and McGonagle, 2012; Ackerman, 2015), well-identified studies are scarce. A number of studies examine the impact of purely vocational education, but fewer compare more or less vocational tracks within regular secondary schools." Duflo, Dupas, and Kremer 2017, Pgs. 1-2. Evans and Yuan 2017 also discusses the lack of evidence linking test scores to long-run gains in human capital. "We estimate the labor market value of improved test scores, assuming that increased learning from interventions corresponds to a long-term human capital gain. This is a strong assumption. Most impact evaluations of education interventions measure impacts over only a short period; McEwan (2015) found the average period between treatment and follow-up measurement across 70 instructional evaluations was 13 months. In some cases, where impacts have been measured over time, the effects have been sustained (Ou 2005; Muralidharan 2012); in others, the effects have diminished or disappeared (Andrabi and others 2011; Jacob and others 2010). There are too few long-term evaluations to draw strong conclusions. As such, this exercise seeks to translate the potential long-term impact of human capital gains into broadly understandable metrics – increased earnings – without intending to be strictly predictive, given the uncertainty of the time path of returns." Evans and Yuan 2017, Pg. 6. "Follow-ups are usually conducted after an academic year of exposure to treat- ments (Table 3). Despite the appeal of estimating longer run effects, it is rare that follow-ups occur more than a month after the treatment ends (Table 3)." McEwan 2015, Pg. 368. Table 3 indicates that of the 70 papers included in the review, only 10% collected data more than a month after the intervention ended. We are aware of the large literature that runs 'Mincerian' regressions that use time in school (typically years of educational attainment) to predict income. For an overview and varying interpretations of this literature, see Card 2001 and Pritchett 2001. For a recent effort at synthesizing estimates of the income returns to education, see Montenegro and Patrinos 2013. 

 5. "Governments in developing countries, international aid agencies, and almost all economists agree that an educated populace is necessary - though not necessarily sufficient - for long-run economic growth and, more generally, a high standard of living. The governments in these countries spend approximately one trillion dollars each year on education, and households spend hundreds of billions more (a precise amount is difficult to calculate) on their children’s education." Glewwe and Muralidharan 2016, Pg. 654.

 

 6. Potential knock-on effects from higher wages at the individual level might include more investment in health and increased financial security, and increased growth of countries may lead to increased provision of public goods and enhanced peace and security, among many other hypothesized effects. "Many see secondary education as having potentially transformative economic and social impacts, particularly for girls. Yet others have more negative views; some experts believe that rapidly expanding access to secondary education will produce little additional learning, given weaknesses in the school system (e.g., Pritchett, 2001). Another hypothesis is that young people see secondary education as promising access to tertiary education and ultimately a government job, with associated rents, and that since such jobs are inherently limited, rapidly expanding education may lead to a cohort of “over-educated” young people, frustrated in their aspirations, and to associated social and political tensions (e.g. Krueger and Maleckova 2003; Heckman, 1991). A third hypothesis is that expanding access to secondary school in developing countries will require curricular changes to prepare students for the labor market. When the United States moved from a system of secondary schools designed to prepare elites for tertiary education to a system of mass secondary education, many secondary curricula dropped Greek and Latin and incorporated vocational education (Goldin, 1999)." Duflo, Dupas, and Kremer 2017, Pg. 1. 

 7. "Although there are strong claims about the effects of secondary education for girls, especially on reproductive health, fertility, and empowerment (UNGEI, 2010; Warner, Malhotra and McGonagle, 2012; Ackerman, 2015), well-identified studies are scarce." Duflo, Dupas, and Kremer 2017, Pg. 2. "Scholarships dramatically changed women’s fertility and marriage outcomes. At age 25, treatment women are 9.1 percentage points (26% of the control mean) less likely to have ever lived with a partner. They are 10.7 percentage points (18%) less likely to have ever been pregnant, 11.5 percentage points (18%) less likely to have had an unwanted pregnancy and have had .217 (27%) fewer children. The effects are seen across the two types of major." Duflo, Dupas, and Kremer 2017, Pg. 25. 

 8. “The conventional wisdom, since at least the writings of John Dewey (1916), views high levels of educational attainment as a prerequisite for democracy.” Acemoglu et al. 2005 

 

 9. “We estimate the effect of education on participation in criminal activity using changes in state compulsory schooling laws over time to account for the endogeneity of schooling decisions. Using Census and FBI data, we find that schooling significantly reduces the probability of incarceration and arrest.” Lochner and Moretti 2004 

 

 10. For example, see Dedvukaj 2016

 

 11. "Formal education in Ghana begins with two years of kindergarten, six years of primary school, and three years of junior high school (JHS). Primary and junior high school are free and enrolment rates are close to 95% in primary school and around 75% in junior high. At the end of JHS, students take the Basic Education Certification Examination (BECE) and those with high enough grades qualify for senior high school (SHS). Passing rates are low. As mentioned above, around 70% of JHS entrants go on to take the BECE and 60% of BECE takers pass. About 20% of those admitted do not enroll in SHS the following year (Ajayi 2014) and many cite costs as the reason. In 2011, government-approved tuition fees for day (non-boarding) students in senior high school were around 500 Ghana cedis per year, a very large sum in a country where the per capita GDP that year was 2400 Ghana cedis." Duflo, Dupas, and Kremer 2017, Pg. 6.

 

 12. "The sample frame for the study was constructed as follows. First, 5 out of the 10 regions in Ghana were included in the study. 10 Across these 5 regions, 54 out of the 170 districts in Ghana were selected because they had a high ratio of day students to boarding students (according to statistics from earlier years), and did not include the regional capital. We focused on day students for budget reasons and because as SHS becomes more common we expect more students to be attending day schools. Across these 54 districts, we selected a total of 177 publicly funded SHS that accept day students." Duflo, Dupas, and Kremer 2017, Pg. 8-9. "To be considered eligible for the study, students needed to satisfy the following criteria: (1) To have been placed into one of the 177 study SHS by the Computerized School Selection and Placement System (CSSPS); (2) To have attended a Junior High School (JHS) in the same district (referred to as “in-district students”) as the SHS they were admitted to; (3) To have not yet enrolled in any SHS by October 2008 (the school year had started in September)." Duflo, Dupas, and Kremer 2017, Pg. 9. "Through visits to both senior and junior high schools, and various interviews with headmasters, teachers and other students conducted in October 2008, we identified 2,246 students eligible for the study. We also asked students why they did not enroll. 95% cited financial difficulties as the main reason, 2% cited pregnancies and 3% cited a variety of other reasons such as being injured, having a job or not liking the school they were placed in. Because students, headmasters and surveyors were unaware of the availability of scholarship at the time of initial surveying, we avoid problems of self-selection into the study sample." Duflo, Dupas, and Kremer 2017, Pg. 9. "In early January, 2009, the 2,246 eligible students were called back to assess whether the student had enrolled or intended to enroll in an SHS for the second term of the 2008-2009 school year. A total of 182 students who either had enrolled or intended to enroll in SHS in the immediate term were dropped from the sample prior to randomization. The final study sample is thus composed of 2,064 individuals (1,028 males and 1,036 females). Among the females, 746 had taken the JHS finishing exam in 2008 only and 290 had first taken it in 2007." Duflo, Dupas, and Kremer 2017, Pgs. 9-10. "The scholarship covered the full tuition and fees for a “day” student for four years." Duflo, Dupas, and Kremer 2017, Pg. 10. 

 13. "The scholarship was paid directly to the school and covered the entire school bill. A typical SHS bill for a day student is comprised of three items: government approved fees which are applied for all schools, PTA (Parents-Teachers Association) dues, and other levies and supplies, including exam fees. The latter two costs are school-specific. In addition to paying school fees, the scholarship also included payment for the final secondary school exam fee (WASSCE). Students who received the scholarship were only responsible for the cost of school materials, the cost of transportation to the SHS and feeding costs (plus boarding costs if they chose to board). The total amount paid by the scholarship program varied slightly across courses and school, but averaged 915 Ghana cedis per student who completed SHS." Duflo, Dupas, and Kremer 2017, Pg. 10. "The cost of SHS school fees for four years in nominal terms was GHX 915. If we assume the real cost of school fees is constant across the four years and adjust for inflation using the CPI (World Bank, 2016), then the average yearly school fees would be 480.36 in 2016 GHX." Duflo, Dupas, and Kremer 2017, Pg. 29. See Exchange-Rates.org website, "Ghanaian Cedis (GHS) to US Dollars (USD) exchange rate for June 30, 2016" 

 14. "To generate high follow-up rates, mobile phones were distributed at the onset of the study to every youth, and study participants were (and still are) sent mobile phone credit twice a year, as an incentive for them to keep the phone number we have on file active. Once a year, we attempt to reach all respondents in order to update their contact information. If they cannot be reached over the phone, we attempt to find them in person by going to their home area. In 2016, 8 years after the start of the study, we were able to reach and interview over 91% of our study sample by phone." Duflo, Dupas, and Kremer 2017, Pg. 11.

 

 15. "By 2016, 74% of the scholarship winners had completed SHS, compared to 47% of the non-winners (Table 3). Thus, while a substantial share of those in the control group was able to put together, over time, the funding necessary to enroll, the scholarship program generates a large gap in educational attainment between winners and non-winners. Winning a scholarship increases the total time spent in SHS by 62% for men and 76% for women (Table 3). Note that repetition is extremely rare, affecting only 1% of students." Duflo, Dupas, and Kremer 2017, Pg. 15.

 

 16. Estimating the effect of being assigned to treatment (the 'intent-to-treat' effect) is relatively straightforward and essentially involves comparing the outcomes of the treatment and control groups. However, to estimate the effect of an additional year of schooling, the authors use the random allocation of scholarships as an instrument for educational attainment. This approach requires an additional assumption, essentially that assignment to a scholarship only affects an individual's later life outcomes through its effects on educational attainment (and not, for example, by directly affecting the short term wealth of their family). The authors explain this approach in more detail in section 4.2 of the paper. They defend the assumptions that it relies upon and argue that although these assumptions might not literally be satisfied, their estimates give a good approximation of the causal impact of education. We think that their approach is reasonable and provides the most robust estimates of the causal effects of education in developing countries that we are aware of. 

 "The effects of free education are of considerable interest in their own right, but they may also shed light on more general issues of the impact of education. In this subsection, we argue that non-educational channels of scholarship effects are likely to be small, and that while exclusion restrictions are probably not literally satisfied, instrumental variable estimates of the effect of education based on using random assignment of scholarship receipt are likely to be reasonable approximations of the causal effect of education." Duflo, Dupas, and Kremer 2017, Pg. 18. "In particular, while the scholarship represented a wealth transfer to infra-marginal families who would have paid for SHS in the absence of the scholarship, it also reduced earnings by children induced to attend SHS by the scholarship during the period of SHS enrollment. We estimate that these effects roughly offset each other in our context, so while we cannot rule out other channels of impact, treating later tertiary education, fertility and labor market effects as due to the effects of the scholarship on education is probably a reasonable approximation." Duflo, Dupas, and Kremer 2017, Pg. 18. 

 17. "Despite the fact that scholarship winners were still more likely to be enrolled in tertiary education by the time they were surveyed in 2016, they were more likely to earn positive income in the last month. Scholarship winners are 5.5 percentage points (s.e. = 2.5 percentage points) more likely to have had any earnings in the past month on a base of 56 percent (Table 6). They are 6.4 percentage points more likely to either have positive earnings or be in school on a base of 63 percent. Overall, they worked 9.97 more hours per month (significant at the 10% level) on a base of 82.7 hours." Duflo, Dupas, and Kremer 2017, Pg. 27.

 

 18. "For students admitted to vocational majors, there was virtually no impact of scholarships on the likelihood that students admitted would be enrolled in formal study/training at the time of the survey. This makes the interpretation of labor market impacts for this group fairly straightforward. We also find no differences by gender, so below we discuss the results for males and females combined." Duflo, Dupas, and Kremer 2017, Pg. 28.

 

 19. "Scholarships increased earnings for this group, with this accounted for by increased hours rather than increased earnings per hour. Scholarships cause a .505 increase in inverse hyperbolic sine earnings. In absolute terms, vocational winners have 25.9 GHX more earnings in the past month than non-winners, a 24 percent increase (significant at the 10% level)." Duflo, Dupas, and Kremer 2017, Pg. 28. The authors use this estimate to calculate the internal rate of return of education at the end of section 6.2, Pg. 29. They note that scholarship winners admitted to vocational majors attended school for an additional 1.19 years. The authors then calculate the additional fees paid for these additional years of schooling as well as the additional costs of schooling (in-kind costs and foregone earnings) for the full four years of senior high school. They assume that scholarship winners admitted to vocational majors earn an additional 25.92 Ghana cedis per month for 30 years and estimate a financial rate of return of 13%. 

 20. "Scholarship winners’ greater earnings are entirely accounted for by additional work hours: scholarship winners work 14.9 more hours per month on a base of 87.0 hours (Table 6, panel B). In turn, the increase in work hours is accounted for by the probability of doing any work – the extensive margin. Winners are 8.8 percentage points (16%) more likely to have any earnings on a base of 56.4% and 11.6 percentage points (22%) more likely to have worked over 10 hours in the past month on a base of 53.8%. Winners do not work significantly more hours conditional on working, they do not earn more conditional on working, and they do not earn more per hour" Duflo, Dupas, and Kremer 2017, Pg. 28.

 

 21. "The increases in employment due to winning a scholarship are concentrated in particular sectors of employment. Winners are 8.5 percentage points more likely to work for a wage. Male winners are much more likely to work as a day or seasonal laborer. Winners are no more likely to work in their own or their family’s business." Duflo, Dupas, and Kremer 2017, Pg. 28. Note that the authors only test for effects on whether individuals work for a wage, work as a day or seasonal laborer, or work in their own or their family's business. They do not test for specific sectors of employment. See Duflo, Dupas, and Kremer 2017, Table 6, Panel C 

 22. "To generate high follow-up rates, mobile phones were distributed at the onset of the study to every youth, and study participants were (and still are) sent mobile phone credit twice a year, as an incentive for them to keep the phone number we have on file active. Once a year, we attempt to reach all respondents in order to update their contact information. If they cannot be reached over the phone, we attempt to find them in person by going to their home area. In 2016, 8 years after the start of the study, we were able to reach and interview over 91% of our study sample by phone. 82% of those who could not be found by phone were then identified in home visits (in total 98% of the study sample were surveyed in just a few months, see Table A1). This is remarkably low attrition for a longitudinal tracking of this kind. Other examples of longitudinal tracking in developing countries have achieved 81% retention over three years (South Africa; Lam, Ardington and Leibbrandt, 2011), 95% (at the household-level) over five years (Indonesian Family Life Survey; Thomas, Frankenburg and Smith, 2002), 91% over seven years (Kenya; Duflo, Dupas and Kremer, 2015) and 84% over ten years (Kenya; Baird et. al 2011)." Duflo, Dupas, and Kremer 2017, Pg. 11. "The labor market section of the callback survey was substantially improved in the 2016 callback. In the 2013 in-person survey and the 2015 callback survey, surveyors asked respondents what their primary occupation/activity was (and if they had one, what their secondary and tertiary activities were) and then asked how much they earned from each of these activities. In the 2013 survey, 56.5% of respondents reported no earnings. In the 2015 survey, 51.7% of respondents reported no earnings. These are primarily respondents who answered “Nothing” as their primary activity, and hence were not asked about secondary activities nor asked about earnings. In follow-up qualitative interviews, respondents revealed that they earn money in ways that they do not consider an “activity” or “occupation”. In the 2016 survey, surveyors asked respondents explicitly if they had any earnings over the past four weeks and how much they earned. In this survey, 43.0% of respondents reported no earnings. The difference in the amount of respondents reporting no earnings between the 2015 data and the 2016 data was greater in the treatment group than in the control group. This may be because secondary school graduates searching for wage employment are less likely to report casual jobs they do here and there as an “activity”. This would suggest that the 2016 data are more accurate, and so most labor market outcomes are analyzed using this data, though we report outcomes from the 2013 and 2015 surveys in Table A3 Panel C." Duflo, Dupas, and Kremer 2017, Pg. 12-13 We note that this income data is self-reported, with a high proportion of individuals reporting no income. We believe administrative data (such as the social security data used for formal earnings in Bettinger et al. 2017) is likely to be less subject to reporting biases. However, in this context, where informal earnings are likely to be important but unlikely to be included in administrative datasets, we think it is appropriate to use well-designed surveys to estimate total income. 

 23. "Among students admitted to academic majors more scholarship winners are still in school. It is thus too early to draw strong conclusions about the labor market impact of scholarships on these students. Nonetheless, we can report some preliminary results. Table 6 columns 4-6 present raw (regression adjusted) differences between labor markets outcomes between winners and non-winners. One variable that is easy to interpret is the effect on having positive earnings or being in school. For the sample as a whole, the point estimate is an insignificant 4 percentage point increase (on a basis of 63%). For females, winning a scholarship increased this by 8.7 percentage points on a base of 50.5% (significant at the 10% level) (Table 6; Panel C); for males the point estimate is negative and insignificant." Duflo, Dupas, and Kremer 2017, Pg. 30. "For students admitted to vocational majors, there was virtually no impact of scholarships on the likelihood that students admitted would be enrolled in formal study/training at the time of the survey. This makes the interpretation of labor market impacts for this group fairly straightforward. We also find no differences by gender, so below we discuss the results for males and females combined." Duflo, Dupas, and Kremer 2017, Pg. 28. The authors estimate upper and lower bounds on the effect of the scholarship on labor market outcomes by making additional assumptions: "If one assumes that the correlation between being induced to attend formal education/training by the scholarship and potential labor market outcomes (if not enrolled) is between 0 and 1, then we can construct an upper bound and lower bound for the treatment effect on the workers who would not be induced to attend further education regardless of whether or not they receive a SHS scholarship (Angrist, Bettinger, and Kremer 2006). The lower bound is simply the point estimate, excluding those who are currently enrolled in formal study or training. The upper bound excludes in addition the top 8.0 percentiles of the labor market distribution in the control for women and the top 2.6 percentiles for men, because those are the percentiles we assume would have been induced to still be in formal school/training by a scholarship, had they been in the control group." Duflo, Dupas, and Kremer 2017, Pgs. 29-30. 

 24. "However, our study also took place in a challenging macroeconomic context in Ghana, and in an environment where, as we describe below, the market was flooded with new graduates: due to a change in the length of secondary school, two cohorts graduated at the same time." Duflo, Dupas, and Kremer 2017, Pg. 5. "The effects we measure should be interpreted as conditional on the macro-economic context at the time, as emphasized by Rosenzweig and Udry (2016). Our study participants began SHS in the 2008/2009 academic year at the earliest. Most participants who completed SHS did so and entered the labor market in July of 2012, and our latest follow-up survey was administered in 2016. Ghana had strong macro-economic performance through the first quarter of 2012, when GDP growth reached an all-time high of 25.0%, but since 2012, GDP growth has fallen each year, reaching a fifteen-year low of 3.92% in 2015. Government policies affecting the labor market also began to shift in 2012. In 2008, the government wage bill was 11.3% of GDP, which was the highest of the 12 West African countries surveyed by the World Bank. The Ghanaian government enacted a new salary scale for government employees in 2012, which raised government wage bill by 38% in one year (IMF, 2012). In 2015, the ballooning wage bill forced the Ghanaian government to impose a net hiring freeze on government employment." Duflo, Dupas, and Kremer 2017, Pg. 7-8.

 "The government also changed their secondary and tertiary education policy during our study period. For the school year 2009/2010, the government shortened the length of senior high school from 4 years to 3 years. The study participants were thus the last cohort enrolled in the four-year program. As a result, most of our participants graduated in a double cohort with the students who had enrolled a year later. In 2014, the government also changed their policy in nursing and teacher training programs. Between the 1980s and 2014, the government paid allowances large enough to cover all fees to all students enrolled in such programs, making them effectively fully subsidized for those admitted, and admissions in the programs were capped via a quota system. Both the allowances and the quotas were removed in 2014, taking into effect for the school year starting in September 2014." Duflo, Dupas, and Kremer 2017, Pg. 8. "Before presenting treatment effects, it is worth noting the bleakness of labor market outcomes in this group. Only 44% of women and 68% of men in the control group earned any money in the month preceding the survey in 2016." Duflo, Dupas, and Kremer 2017, Pg. 26. 

 25. Justin Sandefur made this point on Twitter in a conversation with Pascaline Dupas, one of the authors of the study. 

 

 26. "The effects of free education are of considerable interest in their own right, but they may also shed light on more general issues of the impact of education. In this subsection, we argue that non-educational channels of scholarship effects are likely to be small, and that while exclusion restrictions are probably not literally satisfied, instrumental variable estimates of the effect of education based on using random assignment of scholarship receipt are likely to be reasonable approximations of the causal effect of education.

 In particular, while the scholarship represented a wealth transfer to infra-marginal families who would have paid for SHS in the absence of the scholarship, it also reduced earnings by children induced to attend SHS by the scholarship during the period of SHS enrollment. We estimate that these effects roughly offset each other in our context, so while we cannot rule out other channels of impact, treating later tertiary education, fertility and labor market effects as due to the effects of the scholarship on education is probably a reasonable approximation." Duflo, Dupas, and Kremer 2017, Pg. 18.

 

 27. There is some evidence of negative spillovers in labor market outcomes in an evaluation of a job training program in France (Crépon et al. 2013) and as a result of a school building program in Indonesia (Duflo 2004), though these are very different contexts.

 

 28. “In this view, the (partial equilibrium) distributional implications may thus be very different in different subsamples.” Duflo, Dupas, and Kremer 2017, Pg. 37.

 

 29. "Colombia has comprehensive individual-level administrative data on secondary and tertiary education, female fertility, and labor market experiences. The breadth and depth of the national data provide a unique opportunity to track PACES applicants across a variety of long-run outcomes with little to no attrition in the data." Bettinger et al. 2014, Pg. 13. The authors use five sources of administrative data:

 "To track students’ educational outcomes, we use: 1. The ICFES secondary school graduation/tertiary education entry exam database.... 2. The tertiary education database. We use data from Colombia’s Education Ministry’s Sistema de Prevención y Análisis de la Deserción en Instituciones de Educación Superior (SPADIES) to track scholarship applicants through collegiate pathways, including enrollment and completion. The tertiary education database is an individual-level panel dataset that tracks close to 95 percent of tertiary education students from their first year to their degree receipt beginning in 1998." Bettinger et al. 2014, Pgs. 13-14. "To track government subsidies’ eligibility, informal sector earnings and family formation outcomes we use: 3. The SISBEN Census. We use data from the SISBEN household census of 2010. Data from the SISBEN 2010 survey is used to construct an index score to determine eligibility for government subsidies. SISBEN 2010 covers 57 percent of households in all of Colombia and 39 percent of households in Bogotá." Bettinger et al. 2014, Pg. 14. "To estimate scholarship impacts on earnings we complement SISBEN data with: 4. Colombia’s Social Protection Ministry’s Sistema Integral de Información de la Protección Social (SISPRO). 11 SISPRO is an individual-level panel dataset that is updated monthly, and that contains information on contributions to government social programs for health, employment, and retirement. For the purposes of this study we focus on the work module, which contains information on whether individuals have worked in the formal sector, the number of days of formal sector employment, monthly earnings, and social security contributions. We focus on outcomes from 2008 to 2012 - between seven and 11 years after on-time secondary school completion of scholarship applicants in the Bogotá 1995 sample - since SISPRO only began to cover the universe of formal sector workers in 2008. On average, scholarship applicants would have been around 30 years old at the end of this period. We use SISPRO data to examine additional outcomes, including: extensive and intensive margins of formal sector employment, formal sector earnings and payroll taxes." Bettinger et al. 2014, Pgs. 14-15. "To track consumer credit outcomes we use: 5. Colombia’s financial comptroller’s (Superintendencia Financiera) formal credit census. We focus on outcomes from 2004 to 2014, ten to 20 years after the lottery and four to 14 years after on-time secondary school completion of scholarship applicants in the Bogotá 1995 sample. These quarterly data contain formal credit information for more than 250 million consumer credits, including credit cards and car loans. We focus on two extentive margin outcomes: access to credit card and to car loans, which we define as showing up in the credit data for these loan types. We also analyze credit risk, as measured by interest rates charged on loans." Bettinger et al. 2017, Pg. 14. 

 30. "Current annual formal earnings for scholarship lottery losers are, on average, $2,470 (including zeros). Scholarship lottery winners earn an additional $196 in formal annual earnings, an 8 percent increase (Panel A of Table 4). The p-value on this difference is 0.06.

 Current annual formal earnings for scholarship lottery losers who applied to vocational schools are, on average, $ 2,568 (including zeros). Scholarship lottery winners from applicants to vocational schools earn an additional $427 in formal annual earnings, a 17 percent increase (column 6, Table 4). The scholarship impact difference across academic and vocational applicants for annual formal earnings has a p-value of 0.08 (Panel A, Column 7, Table 4). The effects among vocational school applicants are particularly strong for men. Male lottery losers earn $2,743 while winners earn $535.3 more per year, a 20 percent increase. For males, the scholarship impact difference across academic and vocational applicants for annual formal earnings has a p-value of 0.07 (Panel C, Column 7, Table 4). These results are robust to alternative age specifications as well as to excluding application controls (Appendix B Table)." Bettinger et al. 2017, Pg. 18.

 

 31. PACES stands for Programa de Ampliación de Cobertura de la Educación Secundaria, which roughly translates as 'program to extend coverage of secondary education'. "The PACES scholarship program was introduced 1992 as a way of improving secondary school enrollment rates among disadvantaged students. Most poor students in Colombia attend public schools and —especially in large cities— available slots in public secondary schools were limited when the program began in 1992. The program aimed at tapping the excess capacity in private schools by providing scholarships for private secondary schooling among strata 1 and 2 applicants from public elementary schools (King, Laura Rawlings, Marybell Gutierrez, Carlos Pardo, and Carlos Torres 1997)." Bettinger et al. 2014, Pg. 10. 

 32. "While initially the scholarship covered most tuition fees, the government did not increase its monetary value to keep pace with inflation, and by 1998 the scholarship only covered about 56 percent of the tuition of the average participating school. Families made up for the difference (Angrist et al. 2002)." Bettinger et al. 2014, Pg. 10.

 

 33. "In order to receive an award, students needed to have applied and been accepted to a participating private school. Scholarships were awarded by lottery if demand exceeded scholarship availability. Students were between 12 and 13 years of age at the time of application. Renewal of the award through the end of students’ secondary schooling was contingent upon passing grades. The extent to which this conditionality was at all enforced is unclear (Calderón 1996; Ribero and Tenjo 1997)." Bettinger et al. 2014, Pgs. 10-11.

 

 34. "Colombia’s Social Protection Ministry’s Sistema Integral de Información de la Protección Social (SISPRO). SISPRO is an individual-level panel dataset that is updated monthly, and that contains information on contributions to government social programs for health, employment, and retirement. For the purposes of this study we focus on the work module, which contains information on whether individuals have worked in the formal sector, the number of days of formal sector employment, monthly earnings, and social security contributions. We focus on outcomes from 2008 to 2012 - between seven and 11 years after on-time secondary school completion of scholarship applicants in the Bogotá 1995 sample - since SISPRO only began to cover the universe of formal sector workers in 2008. On average, scholarship applicants would have been around 30 years old at the end of this period. We use SISPRO data to examine additional outcomes, including: extensive and intensive margins of formal sector employment, formal sector earnings and payroll taxes." Bettinger et al. 2014, Pg. 15.

 

 35. "Among scholarship lottery losers, we match 80 percent to the SISPRO government records of those paying payroll taxes, implying that 80 percent of losers ever show up in formal sector employment between 2008 and 2014. Point estimates suggest that lottery winners are 1 percent (0.008 percentage points) more likely to ever appear in formal employment records during this period. However, this difference is not statistically significant (Table 3). Match rate correlates do not systematically differ between winners and losers in the full applicant sample or separately by vocational/academic school application status (Table 3)." Bettinger et al. 2017, Pg. 17. For the exact figures, see Table 3, Pg. 46. 

 

 36. "We define formal sector intensity as the average number of months spent annually in formal sector employment. Both scholarship winners and losers spend about 5.5 months per year in formal sector employment. There are no statistically significant differences in formal sector employment intensity in the full sample, separately by vocational/academic school status or by gender (Table 4)." Bettinger et al. 2017, Pgs. 17-18.

 

 37. "Current annual formal earnings for scholarship lottery losers are, on average, $2,470 (including zeros). Scholarship lottery winners earn an additional $196 in formal annual earnings, an 8 percent increase (Panel A of Table 4). The p-value on this difference is 0.06." Bettinger et al. 2017, Pg. 18.

 

 38. "Current annual formal earnings for scholarship lottery losers who applied to vocational schools are, on average, $ 2,568 (including zeros). Scholarship lottery winners from applicants to vocational schools earn an additional $427 in formal annual earnings, a 17 percent increase (column 6, Table 4). The scholarship impact difference across academic and vocational applicants for annual formal earnings has a p-value of 0.08 (Panel A, Column 7, Table 4)." Bettinger et al. 2017, Pg. 18.

 

 39. "The effects among vocational school applicants are particularly strong for men. Male lottery losers earn $2,743 while winners earn $535.3 more per year, a 20 percent increase. For males, the scholarship impact difference across academic and vocational applicants for annual formal earnings has a p-value of 0.07 (Panel C, Column 7, Table 4). These results are robust to alternative age specifications as well as to excluding application controls (Appendix B Table)." Bettinger et al. 2017, Pg. 18. For the exact tertiary education enrollment figures, see Bettinger et al. 2017, Table 2, Pgs. 44-45. 3.9% of vocational applicants that lost the lottery were enrolled in tertiary education as of 2012, compared to 6% of lottery winners. For academic applicants, 3% of lottery losers were enrolled in tertiary in 2012, compared to 6.1% of lottery winners. "We documented earlier that lottery winners are two to three percent more likely to be still enrolled in tertiary education during the periods we analyze. This may limit winners’ current earnings while increasing their future earnings. To bound what the future earnings difference is likely to be between winners and losers once the former complete tertiary education, we can assume that in the absence of a scholarship, earnings of applicants who attend and complete tertiary education are at the top of the earnings distribution. Under this assumption, we can get an estimate for the effect of winning a scholarship on future earnings by trimming the top two to three percent of formal earners in the scholarship loser group. Appendix E Table shows results for this bounding approach." Bettinger et al. 2017, Pg. 19. "After accounting for the additional proportion of lottery winners currently in tertiary education, we estimate that winning a scholarship may increase future earnings and payroll taxes by up to 23 percent in the full sample and by up 30 percent among vocational applicants." Bettinger et al. 2017, Pg. 20. 

 40. "Relative to losers, scholarship lottery winners pay higher annual payroll taxes at around age 33. The OLS scholarship impact on payroll taxes is $55.1, which at the losers’ mean of $695.9 represents an increase of 8 percent (p-value 0.07). The scholarship impact difference across academic and vocational applicants for annual payroll taxes paid has a p-value of 0.07 (Panel A, Column 7, Table 4)." Bettinger et al. 2017, Pg. 19. "In general, we find little evidence suggesting that, in the full sample of applicants or among female applicants, winning the scholarship affects government welfare receipt of Familias en Acción and subsidized health care programs or eligibility for early childhood care (Table 6). Among male applicants, however, winning a scholarship reduces the likelihood of receiving benefits from Colombia’s Familias en Acción conditional cash transfer program by 27 percent (1.7 percentage points relative to the losers’ mean of 6.2 percent). This point estimate is statistically significant at the 10 percent level (Table 6)." Bettinger et al. 2014, Pg. 24. 

 41. The authors acknowledge that there are a number of channels through which the voucher program may have driven effects in the 2014 paper: "The program may have operated through several potential channels. These include allowing more children to attend private secondary schools, allowing those who might have attended private school in any case to attend a wider range of schools, providing income subsidies to families who might have been predisposed to attend private school, changing students’ peers, and/or incentivizing students and schools to avoid grade repetition. It seems unlikely that the entire scholarship effect is due to schools’ lowering standards since we find effects on tertiary education. Additionally, one might not expect the effects at the top of the distribution if the main channel were increased effort by students concerned that they might lose their scholarships by failing a grade. Scholarship impacts on the sub-population of vocational school applicants are comparable to those in the full sample. The fact that in this subpopulation winners attend schools with peers who are less desirable on observables casts doubt on the notion that peer quality was the only mechanism driving observed scholarship effects (Bettinger, Kremer, and Saavedra 2010)." Bettinger et al. 2014, Pg. 4.

 

 42. "Scholarship lottery winners are 17 percent (7.6 percentage points) more likely to complete secondary school on time relative to the loser’s on-schedule completion rate of 45.2 percent (Panel A of Table 2). Point estimates of scholarship effects are slightly larger (both in percent and percentage point terms) among applicants to vocational schools. Scholarship lottery winners are 10 percent (5.4 percentage points) more likely to complete secondary school within six years after on-schedule completion relative to a base rate of 56.5 percent. During the six years following on-schedule completion, the difference between the proportion of scholarship lottery winners and losers who have completed secondary school declines with each year." Bettinger et al. 2017, Pg. 15.

 The effects on secondary school attainment are not reported in Bettinger et al. 2017 but are reported in Bettinger, Kremer, and Saavedra 2010 in Table 6, Pg. F221. "In prior research on the effects of Colombia's voucher programme, Angrist et al. 2002 find that after three years, lottery winners were 15 percentage points more likely to have attended private school, had completed 0.1 more years of schooling and were about 10 percentage points more likely to have finished 8th grade, primarily because they were less likely to repeat grades." Bettinger, Kremer, and Saavedra 2010, Pg. F211. "The signs of the coefficients suggest uniformly that voucher winners at both types of schools are more likely to complete more years of schooling and less likely to repeat grades." Bettinger, Kremer, and Saavedra 2010, Pgs. F221-222. 

 43. These results are taken directly from Table 2 of Bettinger et al. 2017, Pgs. 44-45 and are discussed in section 4.2 of the paper. "Effects on tertiary education outcomes are particularly strong among students who applied to vocational schools. In this population, the base rate of ever enrollment in tertiary education is 19 percent and this increases by 7 percentage points (37 percent) among scholarship lottery winners. The scholarship impact difference across academic and vocational applicants in the probability of ever enrolling in tertiary education is statistically significant (Column 7, Table 2).1 This ever enrollment effect takes place both in vocational colleges and in universities. Within this group, the effects are particularly driven by males for whom there is a 10 percentage point gain in ever enrollment in tertiary education on the base of approximately 16 percent (Appendix A Table)." Bettinger et al. 2017, Pg. 16. "Among vocational school applicants there is also evidence of gains in tertiary graduation rates and in total years of tertiary education.1 Vocational scholarship winners are 2.4 percentage points more likely to graduate from tertiary education from a base rate of 4.9 percent among vocational scholarship losers. In this population, winners complete 0.19 additional years of tertiary education, which corresponds to a 45 percent increase relative to the base rate of 0.42 years among losers.2 For tertiary graduation and additional years of tertiary education, the scholarship impact difference across academic and vocational applicants is statistically significant (Column 7, Table 2)." Bettinger et al. 2017, Pg. 16. 

 44. "Among applicants to academic secondary schools, there is no evidence of an increase in ever enrolling." Bettinger et al. 2017, Pg. 16. For more detail, see Table 2 of Bettinger et al. 2017, Pgs. 44-45. 

 45. "Quantile regression results indicate that the effects of the scholarship on total formal sector earnings at age 33 are strongest at the top 40 percent of the distribution, particularly the 90th quantile. These effects on the top quantiles are concentrated among vocational school applicants (Figure 1). In terms of formal job characteristics, while there are no differences between winners and losers in firm size, scholarship winners, are more likely to work in growing firms (as measured by new jobs) and firms that pay higher wages (Appendix D Table)." Bettinger et al. 2017, Pg. 19.

 

 46. "Among applicants to vocational schools, scholarship winners attended schools where students were 33 percent more likely to drop out before completing secondary school and were 25 percent less likely to attend college. Despite not having observably more desirable peers, among those who applied to vocational schools, scholarship lottery winners had significantly better educational outcomes than losers, including a 25 percent increase in the likelihood of graduating from high school and a one-third of a standard deviation increase in college entrance examination scores (Bettinger, Kremer, and Saavedra 2010)." Bettinger et al. 2017, Pgs. 10-11.

 

 47. "The scholarship program combined elements of a private school scholarship program with elements of a merit scholarship program insofar as renewal of the scholarship was conditional on grade progression. However, as noted, it is not clear how strongly the later requirement was enforced in practice. If the effects of the program were solely due to its merit scholarship component, then one would expect the strongest impacts to occur among those who are near the boundary of failing grades. In fact, it seems that many of the strongest impacts are at the top of the distribution, such as on tertiary enrolment—which only 19 percent of lottery losers ever accomplish—and on tertiary graduation—which only 5 percent of losers accomplish. Effects on formal sector earnings are also relevant at the top of the distribution. Moreover, we do not observe any effects on the fraction of applicants who are eligible to receive government subsidies. The main place we see an effect that might be at the bottom of the distribution is on teen fertility." Bettinger et al. 2017, Pgs. 34-35.

 

 48. "Because it was administratively difficult to retain the scholarship if one switched schools, there was considerable stickiness in schools attended by scholarship winners. Less than 20 percent of students that transferred after the first year were able to retain their scholarship. Thus, among applicants who applied to vocational private schools, scholarship lottery winners were more likely to stay in vocational schools whereas applicants who did not win a scholarship were more likely to attend academic schools (Bettinger, Kremer, Saavedra 2010)." Bettinger et al. 2017, Pg. 10. 

 

 49. "There are two competing impacts of the scholarship on public expenditures. First, for students who would have attended private school in the absence of the program, the scholarship increases public expenditure. For example, a substantial proportion (87.7 percent) of lottery losers attended private school in sixth grade... Note that the proportion of applicants who attended private school among lottery losers quickly deteriorated (53.9 percent by 8th grade)... From prior data, we know that 54 percent of lottery losers were attending private school in 8th grade and that 32 percent of them finished 11th grade in private school." Bettinger et al. 2017, Pg. 27.

 

 50. If voucher lottery losers who applied to vocational schools were significantly less likely to attend private schools, the policy would have represented a larger income transfer to vocational applicants who won the lottery. However, Table 3 in Bettinger, Kremer, and Saavedra 2010 (Pg. F214) shows that the proportion of lottery losers attending private schools was virtually identical for vocational and academic applicants in the first, second, and third years of the policy. 

 

 51. "Scholarship lottery winners are 17 percent (7.6 percentage points) more likely to complete secondary school on time relative to the loser’s on-schedule completion rate of 45.2 percent (Panel A of Table 2). Point estimates of scholarship effects are slightly larger (both in percent and percentage point terms) among applicants to vocational schools.

 Scholarship lottery winners are 10 percent (5.4 percentage points) more likely to complete secondary school within six years after on-schedule completion relative to a base rate of 56.5 percent. During the six years following on-schedule completion, the difference between the proportion of scholarship lottery winners and losers who have completed secondary school declines with each year." Bettinger et al. 2017, Pg. 15.

 

 52. Table 2 of Bettinger et al. 2017 (Pg. 44-45) shows that 25.8% of vocational applicants that won the voucher lottery have ever enrolled in tertiary education, compared to 19.4% of academic applicants, and finds that this difference is statistically significant at the 5% level. However, approximately 6% of both groups were still enrolled in tertiary education as of 2012, suggesting that a greater proportion of academic applicants either delayed tertiary education or took longer to complete it. 

 

 53. "One plausible hypothesis is that private vocational education is more responsive to labor market and advanced training opportunities than is public education. We find some empirical support for this hypothesis. Among vocational school applicants, effects are particularly strong and precisely estimated for applicants to schools with a commercial focus, even though we cannot reject equality of effects across applicants to different vocational curricula." Bettinger et al. 2017, Pg. 5. Appendix C shows that the effects of the scholarship on formal sector earnings are almost entirely driven by males who applied to vocational schools with a 'commercial' curriculum, with formal sector earnings 58% larger for this group (relative to scholarship losers within this group). The authors note that private vocational schools are more likely to teach a commercial curriculum compared to public vocational schools, which are more likely to have an industrial curriculum: "Within the category of vocational schools, public schools are more likely to teach industrial as opposed to commercial subjects Of public schools, 25 percent have an industrial curriculum and 62 percent have a commercial one, whereas only 4 percent of private vocational schools have an industrial curriculum and 92 percent have a commercial focus (Bettinger, Kremer and Saavedra 2010)." Bettinger et al. 2017, Pg. 8. "Curricula of vocational and academic secondary schools are the same for lower secondary grades (grades 6 through 9). The focus of vocational and academic secondary schools differs for grades 10 and 11—the last two grades of the Colombian secondary school cycle. While academic schools in the last two grades of secondary school focus instruction in the fields of science, humanities or the arts and traditionally prepare students for university education, vocational schools typically, although not exclusively, prepare students for admission into vocational colleges or for participation in the labor market. They typically focus on commercial, industrial, agrarian or pedagogical skills and their curricula exhibit considerable heterogeneity." Bettinger et al. 2017, Pg. 8. "With a single experiment, it is impossible to fully disentangle the channels of program impact. However, gains at the tertiary education level and the labor market, particularly among the sub-population of vocational school applicants, suggest that the impact of the program on secondary completion was not simply due to schools gaming of the system by lowering the standards for grade progression. Instead, our results suggest that private vocational education may improve long-term outcomes by helping students to more effectively transition from secondary school into advanced training and the labor force." Bettinger et al. 2017, Pg. 34. 

 54. "In general, academic schools are more prestigious than vocational schools in Colombia and their students are more likely to complete secondary school and obtain high examination scores. Therefore, among applicants to vocational schools, voucher winners did not attend schools with higher average scores or higher participation rates on Colombia's college entrance examination than their counterparts among voucher losers. In fact, point estimates suggest that among applicants to vocational schools, voucher winners attended schools where students were 25% less likely to attend college and about 33% more likely to drop out. We perform a number of comparisons across multiple measures of peer quality and we find that among applicants to vocational schools, voucher winners attended schools with peers with less desirable observable characteristics than voucher losers." Bettinger, Kremer, and Saavedra 2010, Pg. F205. In addition to the effects of winning a scholarship on tertiary education and formal sector earnings being much larger for applicants to vocational schools (and the differences being statistically significant at the 5% and 10% level respectively), we note that lottery losers that applied to vocational schools earned more than those who applied to academic schools. Vocational applicants who lost the lottery earned an average of $2568 in the formal sector, while academic applicants who lost the lottery earned only $2463 (difference not statistically significant). See Tables 2 and 4 in Bettinger et al. 2017, Pgs. 45, 47. 

 55. "To estimate scholarship impacts on earnings we complement SISBEN data with: 4. Colombia’s Social Protection Ministry’s Sistema Integral de Información de la Protección Social¬ (SISPRO).1 SISPRO is an individual-level panel dataset that is updated monthly, and that contains information on contributions to government social programs for health, employment, and retirement. For the purposes of this study we focus on the work module, which contains information on whether individuals have worked in the formal sector, the number of days of formal sector employment, monthly earnings, and social security contributions. We focus on outcomes from 2008 to 2014 —between eight and 14 years after on-time secondary school completion of scholarship applicants in the Bogotá 1995 sample—since SISPRO only began to cover the universe of formal sector workers in 2008. On average, scholarship applicants would have been around 33 years old at the end of this period. We use SISPRO data to examine additional outcomes, including: extensive and intensive margins of formal sector employment, formal sector earnings and payroll taxes." Bettinger et al. 2017, Pg. 14. "Unfortunately, we do not have data on informal earnings in the full sample, and the data we have is on an endogenously selected subsample allowing us to create only bounds and not point estimates for the impact on earnings within a subpopulation." Bettinger et al. 2017, Pg. 20. 

 56. "Given that there are no effects on the extensive or intensive margin of formal sector employment, these earnings impacts suggest that—through their effects on various educational outcomes—scholarships may have raised productivity. Since we observe no change in formal sector hours, there is no particular reason to believe that increased formal sector earnings are due to a substitution of time away from the informal sector." Bettinger et al. 2017, Pg. 20. "The SISBEN survey covers low-SES neighborhoods and includes about 52 percent of the scholarship applicant population fifteen years after initial scholarship award (Table 6) SISBEN 2010 earnings are a cross-section of self-reported earnings for 2010. Lottery winners are 5 percent (2.8 percentage points) less likely to ever appear in SISBEN data, indicating that they are less likely to reside in poor neighborhoods fifteen years after initial scholarship receipt. This difference is statistically significant at the 10% level (column 1, Table 6). The implication for bounding earnings effects of the scholarship is that adding in the approximately 5 percent of winners who moved out of the low-income SISBEN neighborhoods due to receiving the scholarship would likely increase reported earnings in the SISBEN, making the raw difference between SISBEN earnings among winners and lowers a lower bound on the voucher effect. We estimate an upper bound by trimming the top 5 percent of earners among losers.

 Table 7 reports bounds on the scholarship effect on self-reported total annual earnings from the SISBEN census of the poor. Over two thirds of SISBEN respondents report not paying payroll taxes, which implies that for them these total earnings are informal earnings. The upper bound is $366 on a base of $ 2,000, and statistically significant. The lower bound impact on annual self-reported total earnings is statistically insignificant. Together with the fact that we see no formal labor supply response as a result of winning a scholarship suggests that increased formal earnings are not merely the result of substitution from informal into formal employment among scholarship winners." Bettinger et al. 2017, Pgs. 22-23.

 Table 7 shows lower and upper bounds for the estimated effect of the scholarship on total earnings in the SISBEN data. For the whole sample, the lower bound is negative and statistically insignificant, and the upper bound is positive and statistically significant at the 1% level. However, for applicants to vocational schools, the lower bound is positive but not statistically significant, while the upper bound is positive and statistically significant at the 5% level. Because this sample is not representative, we do not interpret this as strong evidence that overall earnings increased for scholarship winners that applied to vocational schools, though combined with the results on formal sector earnings, we agree with the authors that it is unlikely that increased formal sector earnings are explained by a corresponding decrease in informal sector earnings. 

 57. Note that we are aware of the large literature that uses non-experimental methods to estimate the income returns to education in developing countries, typically by estimating 'Mincerian' regressions of income on educational attainment. For an overview and varying interpretations of this literature, see Card 2001 and Pritchett 2001. For a recent effort at synthesizing estimates of the income returns to education, see Montenegro and Patrinos 2013.

 58. Keats, Anthony. "Women’s Schooling, Fertility, and Child Health Outcomes: Evidence from Uganda’s Free Primary Education Program." May 2016. 59. "One remarkable facet of this project is the variety of course and institution types available to program participants. The TVVP targeted all the major government Village Polytechnics and Technical Training Institutes in the home study area of Busia District, as well as a large cross-section of available private institutions in the area. In general private institutions were eligible to be included in our sample if they had one or more trainees at the time of program recruitment or had offered courses in the prior year, and if their fee structure feel within our voucher limits. Due to the large number and wide range of institutional types in the private vocational schooling sector, the list of potential participating vocational training centers was necessarily far from exhaustive. The most comprehensive list of potential participating institutions was in the primary target area and original home of all of our participants (Busia, Bunyala and Samia Districts). In these areas all formalized private vocational training centers were included. These include for-profit computer training schools and church or NGO-run training centers. Further a variety of privately run for-profit businesses that regularly take students for six month to two year “apprentice-style” training programs were included. These were vetted for legitimacy and formality – shops where space, tools, work and theoretical training were clearly available and where students had been taken many times before were included while those perhaps less equipped to handle a semi-formal training program were excluded. In the rest of western Kenya as well as the cities of Kisumu, Nairobi and Mombasa where some of our sample resided, the program focused primarily on institutions of relatively greater sophistication that more closely resembled public institutions." Hicks et al. 2013, Pgs. 9-10. "Government training institutions under the purview of the MOYAS range from relatively basic village polytechnics, offering traditional self-employment focused industrial trades in skilled construction (masonry, carpentry, plumbing, etc.), automotive mechanics and tailoring, to larger polytechnics in town offering a wider array of courses and complementary skills training in entrepreneurship education (e.g., accounting) and even mathematics. Also included in the partner government institutions are Technical Training Institutes under the Ministry of Education, which offer both industrial education and certain commercial courses in business, computers and secretarial skills. As evidence of the diversity and versatility of the private vocational training sector in Kenya, the type, length and structure of the private institutions and courses in our sample also vary widely. Some institutions run by private entrepreneurs, NGOs or church groups mirror the industrial training structure of the government-run polytechnic system. Others offer short training courses in a particular skill-set like computers or driving. Still others function as businesses and training centers in one, teaching hairdressing, tailoring or some other trade through something akin to an apprenticeship. The private vocational training sector is arguably more adept at accommodating the needs of a larger variety of students, with courses as short as one month well-suited to those already in the work force or supporting their families, to the longer service-based courses desirable to recent secondary school leavers." Hicks et al. 2013, Pgs. 10-11. 

 60. Hicks et al. "Vocational Education in Kenya: Evidence from A Randomized Evaluation Among Youth." The Society of Labor Economics, Aug. 2013. 61. Hardy, Mbiti, and McCasland. “Do Apprentices Alleviate Firms’ Labor Constraints? Evidence from a Unique Experiment in Ghana.” International Growth Centre, Mar. 2016. 62. "The Jóvenes en Acción program was a training program for urban young unemployed in Colombia. It was targeted to unemployed youths 18 to 25, who belonged to the poorest population classified in the two lowest levels of a score, called SISBEN, which is used in Colombia to target all welfare programs. The program was implemented in the seven main cities of the country. It began to enroll students in 2002, and, by 2005, it had enrolled 80,000 students. The goals of the program, which we describe in more detail in online Appendix A, were to develop the youths’ occupational skills, increasing their employability and productivity, to promote the private supply of training, and to improve the matches between workers and firms. Jóvenes en Acción consisted of training courses provided by private institutions (Entidades de Capacitación (ECAP)). Each course was expected to train about 30 unemployed youths, chosen by the ECAPs among eligible applicants. For evaluation purposes, in 2005, the ECAPs were encouraged to select more than 30 applicants; the courses were offered to 30 applicants randomly selected from this group.

 The course had to have three main components: classroom training; ¬¬on-the-job training; and the youth’s project of life (Fondo de Inversión para la Paz (FIP) 2001 and Departamento Nacional de Planeación (DNP) 2016). The program also included a small stipend of about US$2.20 per day for trainees without children under seven years of age, and about US$3.00 per day for women with children under seven. In 2005 there were 114 ECAPs offering 441 courses to 26,615 trainees, with their instructors teaching about 7.6 hours per day." Attanasio et al. 2017, Pgs. 133-134

 "For the purpose of evaluating the intervention by random assignment to treatment, the ECAPs [Entidades de Capacitación, private institutions that provided the training] were asked to select up to 50 percent more applicants than the places available for the courses for the 2005 cohort of trainees. Since the ECAPS were only paid for those who completed the training, they had strong incentives to select individuals accordingly. Two-thirds of the selected applicants were then randomly assigned to the program they applied for, while the remaining one-third were assigned to a control group." Attanasio et al. 2017, Pg. 134. 

 63. "The program raises earnings and employment for women. Women offered training earn 19.6 percent more and have a 0.068 higher probability of paid employment than those not offered training, mainly in formal-sector jobs." Abstract, Attanasio, Kugler, and Meghir 2011. "The follow-up interviews were carried out between August 2006 and October 2006 or between 13 and 15 months after the conclusion of the program." Attanasio, Kugler, and Meghir 2011, Pg. 194. "Table 4A presents treatment effects on employment and earnings for women. Panel A reports effects that take into account site-by-course fixed effects, while panel B, in addition, controls for pretreatment characteristics. Employment increases significantly by 6.1 percentage points and paid employment increases by 7.1 percentage points. This reflects into a significant increase in days worked per month and hours per week. Part of the cost of training for the individual is reflected in the lost tenure, which is estimated to be about −1.5 and is significant. In other words, the controls did find jobs earlier, though not much earlier given that treated individuals were in training and thus out of the labor force for six months. Salary earnings increase significantly by nearly COP$40,000, which corresponds to 22 percent control of women’s earnings. The change in self-employment earnings, albeit positive, is small (at COP$2,000) and not statistically different from zero. Panel B shows that all of these effects are slightly smaller when we control for pretreatment characteristics, which is consistent with a successful randomization. The effects on employment and paid employment when controls are added are 5.4 percentage points and 6.8 percentage points, while the effects on hours and tenure are 2.87 and −1.43. The effect on salaries with controls shows an increase of 19.57 percent." Attanasio, Kugler, and Meghir 2011, Pg. 201. "The treatment effects for men are presented in Table 4B. Here, none of the effects are significant at the 5 percent level, except the reduction in tenure of the program participants by about 3 months. This is true whether we condition on pretreatment characteristics or not. Thus, we have no evidence that the program had any employment or earnings effects for males; rather it appears to have cost them in terms of lost earnings. However, given the potential attrition and sample selection biases, we must be cautious with the interpretation of the results for men." Attanasio, Kugler, and Meghir 2011, Pg. 204. "For both men and women, there is a significant impact of the program on working in the formal sector (as opposed to either not working at all or working in the informal sector). Thus, for women much of the gain in employment was into formal jobs. Men seem to have shifted from informal employment to formal employment, but as explained above, we prefer to interpret the effects on men cautiously because of potential biases due to attrition and because of the initial imbalance. It is possible that some of the trainees were kept on by the firms in which they undertook their on-the-job training. For both men and women this shift has also been reflected in significantly higher formal earnings, although only for women has this meant higher average earnings overall." Attanasio, Kugler, and Meghir 2011, Pg. 206. 

 64. "However, we can carry out the long-term evaluation using two alternative sources: first, we can link those in the evaluation sample to administrative data for 2008–2014; second, we can measure some outcomes for the entire experimental cohort, which is almost ten times the size of the evaluation sample, by linking them to administrative records in 2010." Attanasio et al. 2017, Pg. 135.

 

 65. "We evaluate the long-term impacts of a randomized Colombian training and job placement program. Following the large short-term effects, we now find that the program effects persist, increasing formal participation and earnings contributions to social security and working in larger firms. By using a large administrative source we are also able to establish that the program improved both male and female labor market outcomes by a similar amount--a result that was not apparent with the smaller evaluation sample." Attanasio et al. 2017, abstract. "We find that formal earnings are about COL$35,000 higher among the individuals who were randomly assigned to training, which corresponds to a 13.6 percent increase and is significant at the 2.5 percent level. This shows a remarkable persistence of the effects of the program. In terms of pesos, the effect is similar for males and females, at about 35,000 (Table 3), although for women it represents a higher percentage increase: female earnings in the control group are COL$200,000, while those for males are COL$327,000. Thus, the respective percentage increases are 17.5 percent for females and 10.7 percent for males. Once we control for multiple testing the female effect is only significant at the 6.5 percent level, while the male effect is not significant. However, when we turn to the larger sample from the entire cohort we get much more precise results: they confirm the large and significant effect overall and, crucially for the value of the program, we now can establish that there was a large and significant effect for both females and males. While the point estimate for males is substantially larger than that of women, the difference is not significant." Attanasio et al. 2017, Pg. 139 "The increase in formal earnings is driven by an increase in formal sector employment: the probability of working in the formal sector is increased by 4 percentage points (Table 2). In both datasets the effects are highly significant. In Table 3, we find that for females the probability of being a formal employee increases by 5 percentage points (Romano-Wolf ¬¬ p -value of 0.052). For males, the point estimate is similar but not significant in the evaluation data. But again, when we turn to the larger data, we find a significant and larger effect on formal employment for men (5 percent, ¬¬ p-value 0). In both cases it represents an impact of about 12 percent higher probability of being formal with respect to the control applicants." Attanasio et al. 2017, Pg. 139 

 66. "We use administrative data to examine medium and long-term formal education and labor market impacts among participants and family members of a randomized vocational training program for disadvantaged youth in Colombia. In the Colombian program, vocational training and formal education are complementary investments: relative to non-participants, randomly selected participants are more likely to complete secondary school and to attend and persist in tertiary education eight years after random assignment. Complementarity is strongest among applicants with high baseline educational attainment. Training also has educational spillover effects on participants’ family members, who are more likely to enroll in tertiary education. Between three and eight years after randomization, participants are more likely to enter and remain in formal employment, and have formal sector earnings that are at least 11 percent higher than those of non-participants." Kugler et al. 2015, abstract.

 67. "Among this round of newly designed programs, the Juventud y Empleo (JE) program in the Dominican Republic was one of the first to incorporate a randomized evaluation design. A similar program in Colombia, Jóvenes en Acción, also incorporated a randomized design to allow for the evaluation of training (see Attanasio, Kugler, and Meghir 2009). This article summarizes the impacts of Juventud y Empleo on a wide range of labor market outcomes, including employment, hours of work, monthly earnings, and hourly wages." Card et al. 2011, Pg. 268. "Juventud y Empleo was developed and implemented by the Government of the Dominican Republic with financial support from the IADB. During the period from 2001 to 2006 the JE program focused on low-income youths (ages 18–29) with less than a secondary education (i.e., no more than 11 years of completed schooling) who were not enrolled in regular schooling. Special emphasis was placed on enrolling women. The stated objective of the JE program was to increase the likelihood of employment for the lowest-income members of the working-age population by facilitating access to the labor market through training and counseling. According to the program design mandate, this was to be achieved by adapting the nature of training to the demands of local employers (Inter-American Development Bank 1999)." Card et al. 2011, Pg. 272. 

 68. "We report the impacts of a job training program operated in the Dominican Republic. A random sample of applicants was selected to undergo training, and information was gathered 10-14 months after graduation. Unfortunately, people originally assigned to treatment who failed to show up were not included in the follow-up survey, potentially compromising the evaluation design. We present estimates of the program effect, including comparisons that ignore the potential nonrandomness of “no-show” behavior, and estimates that model selectivity parametrically. We find little indication of a positive effect on employment outcomes but some evidence of a modest effect on earnings, conditional on working." Card et al. 2011, abstract. "Our two basic measures of labor market outcomes in the follow-up survey are an indicator for being employed at the date of the survey and labor market earnings in the month prior to the survey in all jobs (which are equal to zero for nonworkers). As shown in rows 15 and 16 of table 3, the realized treatment group had a slightly higher employment rate (0.574 vs. 0.560) and somewhat higher average earnings (3,133 pesos/month vs. 2,677 pesos/month). The 1.5 percentage point difference in employment rates is not statistically significant (t = 0.5), while the 455 peso difference in monthly earnings is significant at conventional levels (t = 2.13)." Card et al. 2011, 282. 

 69. "This paper presents the results of a large-scale randomized controlled trial of a youth training program, estimating treatment effects six years after random assignment, including long-term labor market trajectories of young people. We are able to track a representative sample of more than 3,200 youths at the six-year follow-up. The intervention is prototypical of many skills training programs worldwide, and has been implemented at scale in the Dominican Republic for more than a decade. Our empirical findings indicate mixed results: on the one hand, we document significant impacts on the formality of employment, particularly for men, and impacts for both men and women in Santo Domingo, the capital. The long-term analysis shows that these impacts are sustained and growing over time. On the other hand, there are no significant impacts on average employment; which appears consistent with the low unemployment in countries with high informality and no unemployment insurance. Looking at the local labor market context, the analysis suggests that skills training programs work better in more dynamic local contexts, where there is actual demand for the skills provided." Ibarrarán et al. 2016, abstract.

 

 70. "Young individuals in Santo Domingo, the capital, also benefit significantly in terms of labor earnings. The empirical results therefore suggest that the skills investment of the program may not bring about large overall impacts, but that it does have a significant impact on the probability of being formally employed and on labor earnings in an urban labor market." Ibarrarán et al. 2016, Pg. 5.

 

 71. "This paper describes the Technical and Vocational Vouchers Program (TVVP) in Kenya and provides early results of the intervention. This program – the first of its kind in Africa, to our knowledge – aims to understand the mechanisms through which vocational education can address the widespread problem of youth underemployment in Kenya, using a multi-faceted randomized evaluation design together with an innovative panel dataset. In particular, through randomized provision of vocational training vouchers to program applicants, the TVVP permits an evaluation of the effects of vocational education on formal sector employment and labor market earnings, participation in the informal and agricultural sectors, entrepreneurship decisions, migration (both within Kenya and to neighboring countries), remittances, fertility decisions and other major life outcomes in a sample of over 2,100 Kenyan youth." Hicks et al. 2013, Pg. 2. "The TVVP is a randomized evaluation of a youth vocational education intervention in (primarily western) Kenya. Approximately 2,160 out-of-school Kenyan youths (18 to 30 years old) applied for vocational education tuition vouchers, and a randomly selected half were awarded vouchers. The vouchers were worth approximately US$460, an amount sufficient to fully (or almost fully) cover the tuition costs for most private vocational education programs and government-run rural village polytechnics or technical training institutes." Hicks et al. 2013, Pg. 6. "One remarkable facet of this project is the variety of course and institution types available to program participants. The TVVP targeted all the major government Village Polytechnics and Technical Training Institutes in the home study area of Busia District, as well as a large cross-section of available private institutions in the area. In general private institutions were eligible to be included in our sample if they had one or more trainees at the time of program recruitment or had offered courses in the prior year, and if their fee structure feel within our voucher limits. Due to the large number and wide range of institutional types in the private vocational schooling sector, the list of potential participating vocational training centers was necessarily far from exhaustive. The most comprehensive list of potential participating institutions was in the primary target area and original home of all of our participants (Busia, Bunyala and Samia Districts). In these areas all formalized private vocational training centers were included. These include for-profit computer training schools and church or NGO-run training centers. Further a variety of privately run for-profit businesses that regularly take students for six month to two year “apprentice-style” training programs were included. These were vetted for legitimacy and formality – shops where space, tools, work and theoretical training were clearly available and where students had been taken many times before were included while those perhaps less equipped to handle a semi-formal training program were excluded. In the rest of western Kenya as well as the cities of Kisumu, Nairobi and Mombasa where some of our sample resided, the program focused primarily on institutions of relatively greater sophistication that more closely resembled public institutions." Hicks et al. 2013, Pgs. 9-10. "Government training institutions under the purview of the MOYAS range from relatively basic village polytechnics, offering traditional self-employment focused industrial trades in skilled construction (masonry, carpentry, plumbing, etc.), automotive mechanics and tailoring, to larger polytechnics in town offering a wider array of courses and complementary skills training in entrepreneurship education (e.g., accounting) and even mathematics. Also included in the partner government institutions are Technical Training Institutes under the Ministry of Education, which offer both industrial education and certain commercial courses in business, computers and secretarial skills. As evidence of the diversity and versatility of the private vocational training sector in Kenya, the type, length and structure of the private institutions and courses in our sample also vary widely. Some institutions run by private entrepreneurs, NGOs or church groups mirror the industrial training structure of the government-run polytechnic system. Others offer short training courses in a particular skill-set like computers or driving. Still others function as businesses and training centers in one, teaching hairdressing, tailoring or some other trade through something akin to an apprenticeship. The private vocational training sector is arguably more adept at accommodating the needs of a larger variety of students, with courses as short as one month well-suited to those already in the work force or supporting their families, to the longer service-based courses desirable to recent secondary school leavers." Hicks et al. 2013, Pgs. 10-11. 

 72. "We obtain our outcome measures from Round 3 of the Kenya Life Panel Survey (KLPS) data (KLPS-3), which was launched in August 2011. The full KLPS sample was randomly divided into two halves, each designed to be representative of the whole, to be tracked in two separate “waves” of data collection during the round. 8 Wave 1 data collection ended in December 2012, at which point survey information had been collected for roughly half of the TVVP sample (1,009 individuals). The tracking rate among TVVP individuals included in Wave 1 data collection was 87.1%, an extremely high rate for a longitudinal survey endeavor in rural sub-Saharan Africa. Crucially, there was no significant difference in tracking rates across the voucher treatment and control groups." Hicks et al. 2013, Pg. 13.

 

 73. "In what follows, we present preliminary impacts of the TVVP. These results are preliminary for two reasons. First, at present we utilize the Wave 1 data only, which represents approximately half of the TVVP sample. We intend to update this analysis when KLPS-3 Wave 2 data collection ends in mid-2014 in order to explore full-sample results. Furthermore, it should be noted that TVVP voucher winners who attended vocational training completed their courses sometime between mid-2009 and early 2011. Thus, some individuals had only been out of training for a matter of a few months at the time data collection was launched, making the study of medium term impacts at this time necessarily tentative." Hicks et al. 2013, Pg. 13. "The evidence shown in this paper suggests that vouchers are a potentially effective way of encouraging investment in vocational education in Kenya. The results show that individuals who were awarded a voucher were able to acquire more vocational education, consistent with the notion that fees and credit constrains limit educational investments in this environment. We found no evidence of sectorial shifts away from agriculture, improvements in well-being and migration due to the program. We also found limited evidence on earnings, where we only see statistically significant increases in wages among wage earners." Hicks et al. 2013, Pg. 25. "We examine the impact of the vouchers on the work sector and labor supply of individuals in our sample. Overall we do not find evidence that the program increased the probability of employment. Examining the extensive margin we do not find a significant increase in the probability of “not being idle”. We also do not see a significant decrease in the probability of our broad measure unemployment (which we define as working zero hours in self or wage employment and looking for a job). Surprisingly we do not find any evidence that the program led to a shift out of agriculture. Contrary to our expectations, we actually found a positive (but insignificant) relationship in both the reduced form and the IV between the program and working in agriculture. We also find suggestive evidence of shifts away from wage or self-employment, perhaps into agriculture… We also do not see any significant change in the labor supply of individuals in our sample. However, we find that the program led to a decrease in the probability of full time employment." Hicks et al. 2013, Pgs. 23-24. "Panel C examines the impact of the program on wage earnings. While we do not find a statistically significant impact of the program on wage earnings for the full sample, we do find that the program led to increases in wage earnings for individuals that worked for a wage (i.e. individuals with positive wage earnings and positive hours in the wage work sector). Focusing on the log wage specifications, our reduced form estimates show that the wages for individuals assigned to treatment and worked in wage sector rose by between 26 to 29 log points (see Figure 4 for the changes in the distribution of wages). The corresponding IV estimates show that the Mincerian rate of return was between 34 and 46 percent for an additional year of vocational education, which is a very high rate of return." Hicks et al. 2013, Pg. 24. 

 74. We have previously written about the limitations of self-reported data and think this is more of a weakness in some contexts than in others. We do not have a good understanding of how these measurements were collected in this study, but the sensitive subject matter (sexual and health behaviors) may make this context more prone to misreporting than for other outcome measures.

 

 75. "Winning a scholarship leads to safer health choices (Table 5). Overall, scholarship winners adopt significantly less risky (self-reported) sexual behavior (-0.052 SD on an index of 9 questions, presented in table A7), have a lower index of STI exposure (-0.074 SD), and more preventative health behaviors (0.116 increase on an index questions on three behavior, hand-washing with soap, bednet use, mosquito repellent use). The impacts on self-reported sexual behavior (riskiness index and exposure to STI index) are significant only for men, but for women we observe actual decline in pregnancies and unwanted pregnancies." Duflo, Dupas, and Kremer 2017, Pg. 26. See Table 5 of Duflo, Dupas, and Kremer 2017, Pg. 51, for full estimates and comparisons between academic major and vocational major admits. 

 76. Table 5 of Duflo, Dupas, and Kremer 2017 (Pg. 51) gives the estimates and the comparison means. The effect size on preventative health behaviors looks reasonably small: scholarship winners adopt 1.74 of the three health behaviors on average, compared to 1.62 of the behaviors adopted by scholarship losers, with the most significant change coming through increased protection from mosquitoes with methods other than bed nets (see Table A7, Panel E, Pg. 73). The risky sexual behavior index is made up of 9 questions (listed in Table A7, Panel C, Pg. 72), with the most significant effects coming from: decreased proportion of males who have ever had sex (69% to 61%), fewer sexual partners in lifetime for males and females (2.28 to 1.92), decreased proportion of males that have had sex with a commercial sex worker (1.9% to 0.6%), increased use of contraception during the most recent time having sex among males who have ever had sex (71% to 83%). The index of STI exposure is made up of 7 questions (listed in Table A7, Panel D, Pgs. 72-73), with the most significant effects coming from fewer individuals reporting having had an STI in the past 12 months (9.6% to 7.2%).

 

 77. "Skeptics of secondary education warn of a potentially large cohort of disaffected students, disappointed by the contrast between their expectations going into education and their outcomes coming out. Given their high initial hopes, the relatively low proportion of SHS graduates who went on to tertiary programs, and the difficulties faced by others in finding a higher-paying job that requires a secondary education, a concern is that the scholarship raised hopes and aspirations, and thereby could have generated disappointment and frustration in the years that followed secondary school graduation. This does not appear to be true in general, although the evidence does not point towards a large positive effect either: a satisfaction index (covering life satisfaction, financial satisfaction and a comparison of their life to others) shows a small insignificant positive treatment effect, as does a mental health index (Table 8). Scholarship winners are as likely as losers to think that they can change their life, and that their life is as good as that of others. The only striking result is that among those who have a job, scholarship winners are much less satisfied with it (a decline of -0.279 on a scale that ranges from 1 to 5, SE: 0.081), but also more confident they can get a better one (an increase of 0.059 on an index that ranges from 1 to 5, SE; 0.034)." Duflo, Dupas, and Kremer 2017, Pg. 33. See Panel F of Table A7, Pgs. 73-74 of Duflo, Dupas, and Kremer 2017 for full results on mental health and Table A8, Pg. 78 for full results on life satisfaction. 

 78. "This paper provides evidence on how STI prevalence and teen pregnancy are affected by two leading policy instruments (and their interaction): education subsidies and HIV prevention education focused on abstinence until marriage. In conjunction with the Kenya Ministry of Education, the Kenya National AIDS Control Council, and the nonprofit ICS Africa, we conducted a large randomized evaluation involving 328 schools in Western Kenya to compare the effectiveness of two programs conducted stand-alone or jointly: (i) the Education Subsidy program, which subsidized the cost of education for upper primary school students by providing two free school uniforms over the last three years of primary school; and (ii) the HIV Education program in which three teachers in each primary school received government-provided training to help them deliver Kenya’s national HIV/AIDS curriculum, which, like many other curricula in Africa and some US states, emphasizes abstinence until marriage as the way to prevent infection. We also estimate the impact of the HIV education program augmented with a small add-on component explicitly stressing condoms within the boundaries of the curriculum… The study involved 328 schools in Kenya’s Western Province. All students enrolled in sixth grade in 2003 were sampled for the study and followed for 7 years, from age 13.5 to 20.5 on average." Duflo, Dupas, and Kremer 2015, Pg. 2758.

 

 79. "We assess the short-, medium-, and long-term impacts of these two programs, implemented alone or jointly, on sexual behavior, fertility, and infection with HIV and another STI, Herpes simplex virus type 2 (HSV2), using a panel dataset that covers a cohort of around 9,500 girls and 9,800 boys over 7 years. For both HIV and HSV2, a positive test result at a point in time reflects having ever been infected with the disease… HIV prevalence was extremely low in the sample, so we focus on HSV2 as our measure of exposure to STIs." Duflo, Dupas, and Kremer 2015, Pg. 2758.

 

 80. "However, the education subsidy alone did not reduce the HSV2 infection rate among either girls or boys… The HIV education program implemented alone did not significantly reduce teenage pregnancy, the risk of HSV2 infection, or schooling attainment among either boys or girls… When the two programs were implemented jointly, fertility fell less than when the education subsidy was provided alone, but HSV2 infections fell more (and significantly). Girls who received the combined program were 20 percent less likely to be infected with HSV2 after 7 years (a drop from 11.8 percent to 9.5 percent). There was no significant impact on the HSV2 infection rate among boys." Duflo, Dupas, and Kremer 2015, Pg. 2759.

 

 81. "The results for girls are surprising because the STI and teenage pregnancy results are not aligned. The only program that reduced STI prevalence (the joint program) is not the program that had the largest impact on pregnancy (the stand-alone education subsidy)... These results are, however, consistent with a richer model of sexual behavior with three features which are realistic in our context. First, teenage girls choose not only a level of unprotected sex, but also choose between “committed” partnerships (in which girls have a single partner whom they believe is also committed to them, and will marry them in the event of a pregnancy), and casual sex, in which there is no expectation of marriage. The costs of pregnancy are perceived to be lower in committed than in casual relationships… Second, schooling and pregnancy are incompatible… The third feature is that girls perceive STI risk to be higher in casual than committed relationships, and the government’s HIV/AIDS education program focused on abstinence until marriage strengthens this perception. Since the cost of pregnancy and the risk of STIs are lower in committed relationships, in the model girls have more unprotected sex in committed relationships than in casual relationships." Duflo, Dupas, and Kremer 2015, Pgs. 2759-2760. "While we do not present a formal test of the model, we show that it generates a series of comparative statics consistent with the data. First, education subsidies lead girls to have less unprotected sex (to avoid pregnancy), conditional on choosing either committed or casual relationships, but can also lead some girls to switch to casual relationships, since committed relationships entail a higher risk of becoming pregnant and having to leave school… Second, when the perceived STI risk from casual relationships increases, as with the abstinence until marriage message of Kenya’s national HIV prevention curriculum, unprotected sex within casual relationships decreases. But a change in perceived STI risk from casual sex also causes some girls to shift from casual to committed relationships. Since unprotected sex is more frequent within committed relationships than casual ones, the overall effect on teenage pregnancies and STIs is ambiguous… Finally, when both programs are implemented jointly, girls have greater incentive to avoid pregnancy so they can take advantage of cheaper education, but they also think that casual relationships carry higher STI risk." Duflo, Dupas, and Kremer 2015, Pg. 2760. 

 82. We also understand that Lant Pritchett and Justin Sandefur are working on a literature review of non-pecuniary effects of education which will in part focus on the health effects of schooling. They have an existing policy paper, Oye, Pritchett, and Sandefur 2016, that uses observational data to show that countries with more school participation of girls have lower child mortality rates and that this reduction is larger in countries with better learning outcomes. We have not read this paper in detail and do not place any emphasis on these results because of their use of observational methods, which do not have a good track record of estimating the causal impacts of policies on outcomes.

 

 83. In addition to reviewing the literature, we also spoke to Justin Sandefur, a senior fellow at the Center for Global Development, who is working on a review of the non-pecuniary returns to education with Lant Pritchett. His preliminary view from this work is that the social effects of education, in particular on fertility and marriage, are substantial, even when there is no learning in schools.

 

 84. "Scholarships dramatically changed women’s fertility and marriage outcomes. At age 25, treatment women are 9.1 percentage points (26% of the control mean) less likely to have ever lived with a partner. They are 10.7 percentage points (18%) less likely to have ever been pregnant, 11.5 percentage points (18%) less likely to have had an unwanted pregnancy and have had .217 (27%) fewer children. The effects are seen across the two types of major." Duflo, Dupas, and Kremer 2017, Pg. 25. "Using treatment as an instrumental variable for years of education, we find that increasing combined years of SHS and tertiary education by one year leads to 0.16 fewer births before age 25 and increasing total years of education by one year leads to .17 fewer births (Table A5). While the OLS estimate is slightly higher (0.19 for secondary/tertiary; .20 for total education), they are not significantly different. Osili and Long (2008) estimate that one year of primary education in Nigeria leads to a reduction of 0.26 births before the age of 25. The base birth rate in the Nigerian study was significantly higher, however, at 2.35 births before age 25, against only 0.8 births before age 25 in our context, thus in percentage terms our effect is larger (20-21% vs. 11%). Appendix table A6, which looks at other years, show that these substantial effects have been persistent and significant since 2013 and the point estimates have generally grown over time. These fertility and marriage results are consistent with the results of a randomized experiment that reduced the cost of access upper primary school in Kenya, and found that the onset of childbearing was also delayed, with no-catch up in the two or three years following school exit (Duflo, Dupas and Kremer, 2015)." Duflo, Dupas, and Kremer 2017, Pg. 25. For full results on marriage and fertility outcomes of female and male scholarship winners, see the first five items listed on Table 5, Pg. 51. 

 85. The marriage and fertility results for the 2013 and 2015 rounds of data collection are reported in Table A6, Pg. 70. In 2013 (average age of 22), female scholarship winners were less likely to have lived with a partner (21% to 16%), less likely to have ever been pregnant (45% to 38%), and were less likely to have had an unwanted first pregnancy (39% to 33%). In 2015 (average age of 24), female scholarship winners were less likely to have lived with a partner (41% to 29%), less likely to have had an unwanted first pregnancy (50% to 40%), and had had 0.166 fewer children on average.

 

 86. "Because the great majority of first pregnancies are reported to be unwanted, the decline we see in women is almost exclusively a decline in unplanned, out-of-wedlock pregnancies. The finding that the hazard of childbearing in the treatment group remains lower for scholarship winners once they are out of school suggests that this is not simply due to an “incarceration effect,” postponing fertility for a few years as in Black, Devereux and Salvanes (2008). It is also not simply due to the fact that reducing the cost of secondary education increases the opportunity cost of pregnancy while of school-age." Duflo, Dupas, and Kremer 2017, Pg. 25. "The more likely potential mechanisms posited by the literature for the effects of education on fertility are: (1) increase in the opportunity cost of bearing and raising children (Becker, 1991); (2) the ability to make better choices thanks to better decoding of information (Rosenzweig and Schultz, 1989); and (3) the fact that education may shape/ change preferences for children. Consistent with channel (1), we find that women winners earn more than women non-winners, which presumably increases the opportunity cost of a child. And consistent with channel (2), we find large increases in learning for both men and women, and we also see that scholarship winners are more likely to report adopting other preventative behavior such as bed net use, handwashing with soap and use of mosquito repellent (Table 5). There is some evidence for channel (3) in our sample, but only for females in academic majors, for whom the scholarship reduced desired fertility by age 50 by 0.21 children (a 5.8% decrease) (Table 5)." Duflo, Dupas, and Kremer 2017, Pg. 26. 

 87. "Bounds on fertility effects of winning a scholarship are tight. Scholarship winners are between 18 and 19 percent (between 4.3 and 4.7 percentage points) less likely to have a child during their teenage years relative to the lottery losers’ (untrimmed) mean of 23.4 percent. Among females, winning a scholarship reduces teen motherhood by between 17 and 19 percent (between 6.5 and 7.4 percentage points) relative to a base of 37.7 percent. The incidence of teen fatherhood is low in Colombia in part because women typically have partners that are older. Hence, for males we examine whether they have children with teen partners. Male lottery winners are between 32 and 34 percent (between 5.1 and 5.6 percentage points) less likely to have a spouse or partner who had a child as a teenager relative to a base rate of 16.1 percent (Panel A, Table 9.)" Bettinger et al. 2017, Pg. 25.

 

 88. "These effects are concentrated among applicants to academic schools. Column 4 of Table 9 shows the results for academic applicants while column 6 shows the results for vocational schools." Bettinger et al. 2017, Pg. 25.

 

 89. "We find no evidence that winning a scholarship changed total fertility since both lower and upper bound estimates on the effect of winning a scholarship on total fertility include zero, At the time of SISBEN 2010, in which applicants are about twenty-eight years old, the average scholarship lottery loser has one child. Lower and upper bound estimates on winning a scholarship on total fertility are close to each other and insignificant, although fairly precisely estimated. This pattern of results is consistent with the “delay” hypothesis by which winning a scholarship keeps a student in school longer without necessarily affecting her opportunity cost of time. In addition to the caveat on bounds, another caveat to this finding is that impacts on total fertility may show up later in the potential childbearing years, so fertility gaps may appear later." Bettinger et al. 2017, Pg. 25.

 

 90. "When implemented alone, the education subsidy program significantly reduced primary school dropouts for both boys and girls and delayed the onset of girls’ fertility… the girls’ teen pregnancy rate fell from 16 percent to 13 percent within that time period. This reduction came entirely through a reduction in the number of pregnancies within marriage, and there was no change in the out-of-wedlock pregnancy rate. By year 7, there was still a 7 percent gap in the childbearing rate between girls exposed to the education subsidy program and those in the control group (46 percent versus 49 ­percent). However, the education subsidy alone did not reduce the HSV2 infection rate among either girls or boys" Duflo, Dupas, and Kremer 2015, Pgs. 2758-2759.

 

 91. See Table 3, Pg. 2776, Duflo, Dupas, and Kremer 2015.

 

 92. "Zomba Cash Transfer Program is a randomized ongoing CCT intervention targeting young women in Malawi that provides incentives (in the form of school fees and cash transfers) to current schoolgirls and recent dropouts to stay in or return to school." Baird et al. 2010, abstract.

 

 93. "An average offer of US$10/month conditional on satisfactory school attendance – plus direct payment of secondary school fees – led to significant declines in early marriage, teenage pregnancy, and self-reported sexual activity among program beneficiaries after just one year of program implementation. For program beneficiaries who were out of school at baseline, the probability of getting married and becoming pregnant declined by more than 40 and 30%, respectively. In addition, the incidence of the onset of sexual activity was 38% lower among all program beneficiaries than the control group. Overall, these results suggest that CCT programs not only serve as useful tools for improving school attendance but may also reduce sexual activity, teen pregnancy, and early marriage." Baird et al. 2010, abstract. "Table III shows that the program led to large increases in school enrolment, especially among those who were not in school at baseline. Column 2 of Table III shows that the percentage of initial dropouts who returned to school (and were in school at the end of the 2008 school year) was 17.2% among the control group compared with 61.4% among treatment. Thus, program beneficiaries were 3–4 times more likely to be in school at the end of the 2008 school year than the control group. 21 Since those enrolled in school actually report attending school over 90% of the time, these treatment effects on enrolment do translate into improvements in actual days of school attended… For the stratum containing baseline schoolgirls, i.e. those who were still in school at baseline, while the absolute numbers are smaller (due to high rates of continued schooling among this group), the relative impact is still impressive (column 3). Among the control group, 89.1% of initial schoolgirls were still enrolled in school at the end of the 2008 school year, compared with 93% in the treatment group. Thinking of these as dropout rates, the CCT program reduced the dropout rate among this group by 35% – from 10.9% among controls to 7% among treatments." Baird et al. 2010, Pg. 63. "Table IV presents the impact of the program on having never been married. Early marriage increases coital frequency, significantly decreases condom use, and virtually eliminates the ability to abstain from sex (Clark, 2004). As described earlier, the study sample was selected to be never married at baseline, and so levels of marriage are equal to the incidence during 2008. We see that 27.7% of initial dropouts in the control group have gotten married during the past year, compared with only 16.4% of the same group in treatment (column 2). This is a reduction in the marriage rate of more than 40% among those who were not in school at baseline. However, we also note that the program had no effect on the propensity to get married among the baseline schoolgirls – 4.7% of whom got married both among the controls and treatments." Baird et al. 2010, Pg. 64. "Table V describes the impact of the program on the incidence of childbearing – i.e. the likelihood of ever being pregnant. Column 2 shows that baseline dropouts among the treatment group are 5.1 percentage points less likely to have become pregnant over the past year, a reduction in more than 30% that is statistically significant at the 5% level. Again, as with marriage, the CCT program had no impact on the incidence of childbearing at follow up for baseline schoolgirls." Baird et al. 2010, Pgs. 64. "Finally, we present impacts on self-reported sexual activity and risky behaviors. Table VI (a) examines onset of sexual activity and the number of sexual partners in the past 12 months. At baseline, 29.6% of initial dropouts and 79.4% of initial schoolgirls reported having never had sex. Columns 2–3 of Table VI (a) indicate that the reduction in the onset of sexual activity is 5.5 percentage points among initial dropouts (significant at the 1% level) and 2.5 percentage points among initial schoolgirls (with a p-value of 0.112), which represent reductions in the onset of sexual activity of 46.6 and 31.3%, respectively. Columns 5 and 6 complement this finding and show that the change in the number of self-reported number of lifetime partners from baseline to follow up is smaller for the program beneficiaries." Baird et al. 2010, Pgs. 65. 

 94. "This article assesses the role of conditionality in cash transfer programs using a unique experiment targeted at adolescent girls in Malawi. The program featured two distinct interventions: unconditional transfers (UCT arm) and transfers conditional on school attendance (CCT arm)." Baird, McIntosh, and Özler 2011, abstract.

 

 95. "Although CCTs were more cost-effective than UCTs in increasing school enrollment and attendance, they had little effect on reducing the likelihood of teenage pregnancies or marriages. By contrast, UCT offers were very effective in delaying marriage and childbearing—by 44% and 27%, respectively, after 2 years. These impacts in the UCT arm were experienced almost entirely among those who dropped out of school after the start of the 2-year intervention, whereas the likelihood of marriage and pregnancy were negligible among those who stayed in school regardless of treatment status." Baird, McIntosh, and Özler 2011, Pg. 1747. "Column (1) of Table VII shows that by Round 2, 4.3% of the initially never-married sample was married in the control group. Marriage rates were unchanged in the CCT arm but significantly lower in the UCT arm. By Round 3, the prevalence of marriage rose to 18.0% in the control group with an insignificant reduction of 1.2 pp in the CCT arm and a significant reduction of 7.9 pp (44%) in the UCT arm (column (2)). The differences in program impacts between the two treatment arms in Rounds 2 and 3 are both statistically significant at the 95% confidence level. Columns (3) and (4) show that although the program had little effect on the prevalence of pregnancy in either treatment arm after 1 year, there is a large decline in the UCT arm by Round 3. While roughly a quarter of the control group and the CCT arm had been ever pregnant by the end of the experiment, this likelihood was reduced by 6.7 pp (or 27%)in the UCT arm—significant at the 99% confidence level. The difference in program impacts between the two treatment arms is also significant at the 99% confidence level by Round 3." Baird, McIntosh, and Özler 2011, Pgs. 1735-1736. 

 96. Duflo, Dupas, and Kremer 2017 finds evidence of sustained decreases in fertility rates among female scholarship winners, suggesting that these are not just 'incarceration effects'. In contrast, Bettinger et al. 2017 finds no evidence of reduced fertility at age 30, indicating that 'incarceration effects' might be important. Jensen 2012 estimates that improved labor market opportunities in India increase educational attainment of women and reduce marriage and fertility, indicating that women do consider the opportunity cost of pregnancy and marriage when making these decisions.

 

 97. For example, if the income effects of educational policies are driving the reductions in teenage pregnancy and marriage, then (targeted) unconditional cash transfer programs might be more cost-effective than education programs.

 

 98. Duflo, Dupas, and Kremer 2017 evaluates the effects of a scholarship program, Bettinger et al. 2017 evaluates a school voucher program, Duflo, Dupas, and Kremer 2015 evaluates secondary school subsidies, and Baird et al. 2010 and Baird, McIntosh, and Özler 2011 evaluate cash transfers.

 

 99. It is possible, however, that the estimated effects on fertility and marriage are driven by girls in families that were 'infra-marginal' (i.e. would have gone to school anyway without the scholarship) and who therefore received a transfer of income. The effect is estimated for the overall sample, but it is possible that it is driven primarily by this subgroup and therefore explained by income, not educational effects.

 

 100. "Many see secondary education as having potentially transformative economic and social impacts, particularly for girls… This debate is surprisingly uninformed by high quality evidence from the developing world. Many studies in the developed world have used natural experiments to estimate the rates of return to education (e.g., Angrist and Krueger, 1990). However, it is not clear if the results generalize to developing countries which have vastly greater levels of education than did developed countries when they had comparable income levels (Pritchett, 2001). While many studies document the positive correlation between education and other outcomes, there are surprisingly few well-identified studies from lower income countries on the causal impacts of education. We are aware of no randomized controlled trial (RCT) and only one study based on regression discontinuities in admission test scores on the labor market impact of secondary education (Ozier, 2016). Although there are strong claims about the effects of secondary education for girls, especially on reproductive health, fertility, and empowerment (UNGEI, 2010; Warner, Malhotra and McGonagle, 2012; Ackerman, 2015), well-identified studies are scarce. A number of studies examine the impact of purely vocational education, but fewer compare more or less vocational tracks within regular secondary schools." Duflo, Dupas, and Kremer 2017, Pgs. 1-2. Evans and Yuan 2017 also discusses the lack of evidence linking test scores to long-run gains in human capital: "We estimate the labor market value of improved test scores, assuming that increased learning from interventions corresponds to a long-term human capital gain. This is a strong assumption. Most impact evaluations of education interventions measure impacts over only a short period; McEwan (2015) found the average period between treatment and follow-up measurement across 70 instructional evaluations was 13 months. In some cases, where impacts have been measured over time, the effects have been sustained (Ou 2005; Muralidharan 2012); in others, the effects have diminished or disappeared (Andrabi and others 2011; Jacob and others 2010). There are too few long-term evaluations to draw strong conclusions. As such, this exercise seeks to translate the potential long-term impact of human capital gains into broadly understandable metrics – increased earnings – without intending to be strictly predictive, given the uncertainty of the time path of returns." Evans and Yuan 2017, Pg. 6. "Follow-ups are usually conducted after an academic year of exposure to treat- ments (Table 3). Despite the appeal of estimating longer run effects, it is rare that follow-ups occur more than a month after the treatment ends (Table 3)." McEwan 2015, Pg. 368. Table 3 indicates that of the 70 papers included in the review, only 10% collected data more than a month after the intervention ended. We are aware of the large literature that runs 'Mincerian' regressions that use time in school (typically years of educational attainment) to predict income. For an overview and varying interpretations of this literature, see Card 2001 and Pritchett 2001. For a recent effort at synthesizing estimates of the income returns to education, see Montenegro and Patrinos 2013. 

 101. We have also consulted other reviews for this report, though we have not read these in detail: Evans and Popova 2016, Ganimian and Murnane 2014, McEwan 2015, Glewwe et al. 2012, Banerjee et al. 2013, Kremer, Brannen, and Glennerster 2013, Behrman 2010 and Glewwe and Kremer 2006.

 

 102. "One potential reason why parents invest relatively little in their children’s education is that they, or their children, may not be aware of sizeable returns to education in the labor market. Two high quality studies have examined the impact on students’ time in school of providing this type of information to parents and/or their children. As seen in the first line of Table 4, of the four estimates from these two studies, two (both from the same study) are statistically insignificant while two (one from each study) are significantly positive. The first study of this type was by Jensen (2010), who implemented an RCT that provided information on (estimated) returns to schooling to boys in grade 8 from poor households in the Dominican Republic. This information was provided to these students in the form of a brief statement read to the students at the end of a survey. The motivation for this intervention was the finding that these students generally assumed that the returns to additional years of schooling were very low, which may explain their low rates of secondary school completion. Jensen found that the boys who received the information were 4.1 percentage points more likely to be in school one year after receiving the information (significant at the 10% level). He also found, using data collected four years later, that the boys who received the information were 2.3 percentage points more likely to finish secondary school (not significant) and had completed, on average, 0.2 more years of schooling (significant at the 5% level). The effects were strongest for the least poor of those in poor households, and weakest for the poorest of the poor households, which suggests that the latter may face other barriers, such as credit constraints. In contrast, a similar but more intensive intervention in China by Loyalka et al. (2013) found little effect of providing information. They focused on boys and girls in grade 7 in poor rural areas of two Chinese provinces (Hebei and Shaanxi). As in the Dominican Republic, an initial survey suggested that these students had inaccurate information about the costs and benefits (in terms of future earnings) of education (in particular, they overestimated the cost of vocational education), and that they also lacked career planning skills. The intervention consisted of providing a 45 minute information session on earnings associated with different levels of schooling. The evaluation was implemented as an RCT, and the main result is that the program had no significant effect on the dropout rate. The authors speculate that students felt that the quality of their schools was low, and thus they assumed that for their schools the additional time in school would not produce high returns, but they present no evidence to support this conjecture. Loyalka et al. (2013) also examined whether this intervention increased students’ test scores, and found no significant effect, as seen in the first row of Table 5." Glewwe and Muralidharan 2016, Pgs. 20, 21. 

 

 103. "Two high-quality studies, both from Kenya, have examined the impact of merit-based scholarships on students’ time in school. As seen in the ninth line of Table 4, three of the four estimates from these two studies find significantly positive effects (the fourth estimate is statistically insignificant). The first such study was that of Kremer et al. (2009), who conducted an experimental evaluation of a scholarship program in rural Kenya that targeted girls who were in grade 6 (Kenya’s primary schools teach students from grade 1 to grade 8). At the beginning of the school year, girls in grade 6 were told that those who scored in the top 15% on end-of-year exams would be given, for each of the next 2 years (grades 7 and 8), an amount of money equal to $6.40, which was enough to cover school fees, and in addition their parents would be given an amount of money equal to $12.80. The second study in Kenya, by Friedman et al. (2011), was a follow-up of this 2009 study; it examined the educational outcomes of the same girls 4–5 years after the original program started (about 2 years after the program had ended). The earlier study found that the program significantly increased participation (daily attendance, where girls who leave school have a zero attendance rate) by 3.2 percentage points. The more recent study found that the program had significant positive impacts on enrollment in secondary school (8.6 percentage point increase) and current enrollment in any school (7.9 percentage point increase), but not on grades completed." Glewwe and Muralidharan 2016, Pg. 686.

 

 104. "Consider next the impact of scholarship programs on student learning. Four studies, all randomized controlled trials, have examined the impact of providing scholarships based on student learning as measured by test scores. Their results are shown in the seventh line of Table 5. The findings are almost unanimous: five of the six estimates are significantly positive, while the sixth was statistically insignificant. The two Kenya studies, which were described above, found significantly positive estimates. More specifically, Kremer, Miguel and Thornton (2009) found a 0.27σ increase in scores on a grade 6 year-end exam, and Friedman et al. (2011) found that test scores were 0.20σ higher 4-5 years after the program started (two years after it ended). A more recent study is that of Blimpo (2014), who examined a program in Benin that randomly assigned 100 secondary schools to a control group or to one of three different types of scholarships: scholarships based on individual-level performance with respect to a set goal, with no limit on the number of scholarships offered; scholarships based on average performance for (randomly assigned) teams of four students, again with respect to a set goal and no limit on the number of scholarships; and a “tournament” in which 84 teams of four students each (randomly assigned) from 28 schools competed for a large prize that was given only to the three top performing teams. For the first two types, the payments were $10 per person ($40 for a team of four) for a relatively low level of performance, and $40 per person ($160 for a team of four) for a high level of performance. For the third type, the prizes were much larger, at $640 for each of the top three teams. All three types of incentives had similar (and statistically significant) impacts, increasing grade 10 test scores by 0.24σ to 0.28σ. Finally, a scholarship program in China, evaluated by Li et al. (2014), was also based on a tournament. In one version (individual incentive), groups of 10 low performing students would compete with each other in terms of improvement over time on their test scores. The top student received about $13, the second and third about $6, and the rest each received about $3. In the other version (peer incentive), each of ten low performing students was paired with a high performing student; prizes for the low performing students were the same as in the first version, but in this case the high performing student also received a similar reward in order to encourage that student to assist the low-performing student with whom he or she had been paired.17 The individual incentive intervention in China had no statistically significant impact. However, the authors find that combining student incentives with peer tutoring (where academically higher achieving students were paired with lower achieving ones and both students were rewarded for improvements) increased the test scores of the weaker students by 0.27σ. Thus, in some cases it is possible that student incentives on their own may not be effective unless also accompanied by pedagogical support." Glewwe and Muralidharan 2016, Pgs. 32, 33. 

 

 105. "One way to make schooling more effective, which should in turn increase the demand for schooling, is to provide information to mothers on how to develop their child’s learning. Such an initiative was examined by Banerji, Berry and Shotland (2013), who also examined the impact of a mother literacy program and the combination of both programs. As seen in lines 11-13 of Table 4, neither of these two interventions, nor their combination, had an impact on children’s time in school (as measured by enrollment and daily attendance). Turning to the effects on student learning, which are summarized in lines 9-11 of Table 5, the lessons on child learning led to small but statistically significant increases in the test scores (average over literacy and mathematics) of students in grades 1-4 (0.04σ), but there was no significant impact of the literacy class alone. Interestingly, a somewhat higher impact (0.06σ) on student test scores was found for women who took both classes. These impacts after one year of the program are relatively small, and there were no significant impacts on children’s enrollment rates." Glewwe and Muralidharan 2016, Pg. 34. 

 

 106. "Five high-quality studies have examined the impact of building new schools on time in school. Each of these five studies examined a different country, so evidence is available from Afghanistan, Burkina Faso, Indonesia, Mozambique, and Pakistan. Building new schools reduces a very important indirect cost of attending school, the distance to the nearest school. More time spent traveling to school is time lost for other activities, including work, and greater distances may also lead to direct transportation costs and worries about safety. The first two rows of Table 6 show that there are two studies based on RCTs and three studies that use other high-quality estimation methods. All six estimates from these five studies show significantly positive impacts from building new schools on students’ time in school." Glewwe and Muralidharan 2016, Pg. 692. "Two of these five high-quality studies that examined the impact of building new schools on time in school also examined the impact of that intervention on student learning. As seen in the first two rows of Table 7, these two studies have produced three estimates of this impact. While the number of studies, and the number of estimates, is very small, the findings are unanimous: all three estimates are statistically significant and show that constructing new schools increases learning among children of school age." Glewwe and Muralidharan 2016, Pg. 693. "Perhaps the second most obvious “input” that affects students’ educational outcomes (the first being providing an actual school) is the amount of time that children spend in school. For any given year, this time could be lengthened by increasing the number of hours per day that the school is open or by increasing the number of days per year that the school is in session. We found no high-quality studies that examined the impact of a longer school day on time in school, perhaps because it is rather obvious that longer school days by definition increase time in school, assuming that the rate of student absenteeism does not dramatically increase. However, two high-quality studies have produced four estimates of the impact of an increase in the length of the school day on student learning, as seen in the third line of Table 7. Of these estimates, three are positive and significant and one is insignificant. The positive and significant estimates come from studies of Chile (Bellei, 2009) and Ethiopia (Orkin, 2013), both of which are based on difference-in-differences estimation. The one insignificant, estimate is also from the Orkin study. Overall, the evidence, while based on only two studies, is generally supportive of the common sense notion that longer school days increase student learning." Glewwe and Muralidharan 2016, Pg. 694. 



 107. "Two RCT studies from Sub-Saharan Africa have examined the impact of textbooks on students’ educational outcomes: Glewwe et al. (2009) and Sabarwal et al. (2014).... Both of the two estimates (one from each of these two studies) are statistically insignificant; providing textbooks did not decrease dropping out or grade repetition in Kenya and did not increase daily attendance in Sierra Leone…. While textbooks may not increase students’ time in school, one would expect that they would increase student learning. This is examined in the fourth row of Table 7, which summarizes the results of three estimates from the Kenya and Sierra Leone studies. Surprisingly, none of these three estimates is statistically significant." Glewwe and Muralidharan 2016, Pgs. 694-695. "Another study from the same area of Kenya by the same authors (plus another author), Glewwe et al. (2004), used an RCT to examine the impact of the provision of 'flip charts' (sets of large posters to place on an easel or hang on a wall) on students’ test scores in Kenya. As seen in the fifth line of Table 7, the authors find that the provision of flip charts had no significant impact on student learning, and they note that nonexperimental estimates of the impact of this intervention find significant positive effects." Glewwe and Muralidharan 2016, Pg. 695. "We found only one high-quality study of the impact of libraries on students’ educational outcomes, a study of the provision of school libraries in India by Borkum et al. (2012). The authors used an RCT to examine the provision of both “in school” libraries and traveling libraries. As seen in the fifth line of Table 6, there was no effect of either library on students’ daily attendance (which was already quite high, at around 90%). Turning to learning outcomes (sixth line of Table 7), “in school” libraries had no effect on students’ language scores and the traveling libraries had an unexpected negative effect." Glewwe and Muralidharan 2016, Pg. 695. "The last two pedagogical material interventions are from the same study, which examined primary schools in the Philippines. Tan et al. (1999) experimentally investigated the impact of the provision of “multilevel learning materials,” as well as an intervention that combined those materials with “parent-teacher partnerships.” The multilevel learning materials intervention by itself significantly reduced the probability of dropping out, but had no significant impact on dropping out when combined with parent-teacher partnerships (lines 6 and 7 of Table 6). The impact on test scores (in Filipino, math and English) was significantly positive for two of the three tests when only multilevel materials were provided, and for all three tests when those materials are combined with parent-teacher partnerships (lines 7 and 8 of Table 7). Of course, it is unclear what the relative contributions of two components are. Even the multilevel learning materials intervention by itself had many components (several different types of learning materials), so it is not clear which components led to increased student learning." Glewwe and Muralidharan 2016, Pg. 696. 

 108. "Teachers vary in many ways, but we found no high-quality studies that have examined the impact of teacher characteristics on student learning or time in school. However, one high-quality study has examined the impact of providing an extra teacher to very small primary schools in India, and three studies have examined the impact of variation in the pupil-teacher ratio. All four of these studies can be thought of as attempts to change the 'quantity', as opposed to the 'quality', of teachers." Glewwe and Muralidharan 2016, Pg. 696. "Only one of these studies focused on the impact of the quantity of teachers on students’ time in school. Chin (2005) used a DD approach to evaluate a program that provided extra teachers and additional educational materials (including blackboards) to very small primary schools in India. As seen in the eighth row of Table 6, she found that the program significantly increased students’ primary school completion rates (by 1-2 percentage points), but it is not possible to determine how much of this effect is due to the extra teacher and how much is due to the additional educational materials." Glewwe and Muralidharan 2016, Pg. 696. "Turning to the pupil-teacher ratio, three high quality studies have produced five estimates of the impact of the pupil-teacher ratio on student learning. These results are summarized in the ninth and tenth rows of Table 7. Intuitively, one would expect that increased pupil-teacher ratios reduce learning because larger classes reduce opportunities for teachers to give individual attention to students. Indeed, three of the five estimates, from two different studies, find a significantly negative effect. On the other hand, two of the five estimates, also from two studies, including the only RCT study, are statistically insignificant. The two papers that produce the expected negative finding are those by Urquiola (2006) and Urquiola and Verhoogen (2009). Urquiola (2006) used regression discontinuity methods to estimate the impact of the pupil-teacher ratio on student learning in Bolivia. In particular, he used the fact that schools with pupil-teacher ratios above 30 can apply to the education authorities for another teacher, and he presents evidence that these schools often do obtain another teacher. He finds that schools that obtain another teacher, which greatly reduces the pupil-teacher ratio, have significantly higher language scores, but the effect on math scores is not statistically significant." Glewwe and Muralidharan 2016, Pg. 41. 

 109. "The most common type of program that attempts to improve students’ nutritional status is the provision of school meals. Lines 9 and 10 of Table 6 show the results from five studies in five different countries (Burkina Faso, Chile, India, the Philippines, and Uganda) that have estimated the impact of school meals on time in school. Of the seven estimates, six are statistically insignificant and one, for the program in Burkina Faso that was evaluated by Kazianga et al. (2012), finds (using an RCT) a significantly positive impact on enrollment of children aged 6–15 years old. This evidence indicates that in most cases school meals do not increase students’ time in school. Even if school meals do not increase years of enrollment, one would think that it would increase daily attendance. Yet only one of the five studies, that of Alderman et al. (2012), measured the impact of school meals on daily attendance; the authors found no statistically significant, effect…. Of these five high-quality studies, four have examined the impact of school meals on student learning. As seen in lines 11 and 12 of Table 7, three of the seven estimates from these four studies are statistically insignificant and four are significantly positive." Glewwe and Muralidharan 2016, Pg. 698. "Two studies have examined the impact of take-home rations on students’ education outcomes. The results on students’ time in school are summarized in lines 11 and 12 of Table 6. Adrogue and Orlicki (2013) found no impact of such a program on student attendance in Argentina, but Kazianga et al. (2012) found a significantly positive 4.8 percentage point impact on the enrollment of children aged 6–15 in Burkina Faso. Only the latter program examined the impact of take-home rations on student learning; as seen in line 13 of Table 7, the Burkina Faso program increased students’ math scores by 0.08 standard deviations, an impact that is statistically significant (p-value < 0.05)." Glewwe and Muralidharan 2016, Pg. 698. 

 110. "One of the earliest published RCT studies in education is that of Newman et al. (2002), who examined a program in Bolivia that provided funds to schools to make infrastructure improvements. They estimated the impact of this program on the dropout rates of boys and girls in grade 7. As seen in Table 6, the program had no statistically significant impact on those dropout rates." Glewwe and Muralidharan 2016, Pg. 700. "In a similar vein, in a paper for which the main contribution is methodological, Chay et al. (2005) examine Chile’s P-900 program, which provided infrastructure, materials and teacher training to schools in Chile. They use a regression discontinuity approach to estimate the impact of this program. While two of the four estimates are significantly positive, it is virtually impossible to determine which component of the program led to increased student learning." Glewwe and Muralidharan 2016, Pg. 701. 

 111. "The first rows of Tables 8 and 9 summarize the results from three studies that examined the impact on students’ educational outcomes of interventions that focus on “teaching at the right level,” which typically involves remedial/supplemental instruction and/or tutors or volunteers to provide that instruction. Only one of these studies examined the impact of teaching at the right level on students’ time in school; as seen in Table 8, that study found no significant impact. All three studies considered impacts on student learning, and together they have produced six estimated impacts. As summarized in Table 9, two of these estimates are statistically insignificant while four are significantly positive." Glewwe and Muralidharan 2016, Pg. 703. 

 

 112. "Indeed, an old question in the economics of education is whether students in a given grade benefit from being “tracked” into classrooms based on their initial learning levels or ability. The main argument for tracking is that the reduction in variance in the ability levels of students in the classroom may make it easier for teachers to more effectively match the difficulty level of the content and material they teach to the level of their students. The main argument against tracking is the concern that students who are tracked to “lower” level classrooms may suffer further from negative peer effects and from stereotyping and loss of self-esteem, which may place them on a permanently lower trajectory of learning. A further concern is that some education systems may track students very early using data that may be noisy and not sufficiently reliable for tracking." Glewwe and Muralidharan 2016, Pgs. 706-707. "Fortunately, however, there is one high-quality study of tracking that was conducted in the context of a developing country, that of Duflo et al. (2011). They did not examine the impact of tracking on time in school but, as seen in the second row of Table 9, they examined the impact of tracking on student learning. More specifically, they conducted an experimental evaluation of tracking in Kenya and found that tracking and streaming of pupils appears to have a positive and highly significant effect on test scores in both the short term and the long term. Students in tracking schools scored on average 0.18σ higher than students in nontracking schools, and continued to score 0.18σ higher even 1 year after the tracking program ended, suggesting longer-lasting impacts than those found in many other education interventions… Most importantly, Duflo, Dupas, and Kremer found positive impacts for students at all quartiles of the initial test score distribution and cannot reject that students who started out above the median score gained the same as those below the median; those in higher achieving classes scored 0.19σ higher than the higher achieving control school students, while those in lower achieving classes scored 0.16σ higher than the lower achieving control school students." Glewwe and Muralidharan 2016, Pg. 707. 

 113. "There is much more evidence on the impact of technology-enhanced instruction on student learning; the third and fourth rows of Table 9 indicate that there are 17 estimates from nine different studies. Yet these results point to widely varying magnitudes of impact, perhaps varying more than almost any other intervention considered in this chapter, including estimates that are significantly negative and others that are significantly positive. Further caution is in order because all but one of the studies discussed below are based on RCTs, with high contextual internal validity of the estimated impacts but uncertain external validity. Thus, the differences in estimated impacts are quite striking and point to the importance of context, and perhaps more importantly to the importance of program design in creating effective programs of technology-aided instruction." Glewwe and Muralidharan 2016, Pg. 708. "Our summary of the evidence, as well as the brief discussion of the theoretical mechanisms, suggest that there are many good reasons to be excited about the potential for technology-enabled instruction to produce substantial improvements in students’ learning outcomes. However, the evidence on the impact of greater use of technology in the classroom is mixed, and program impacts seem to depend crucially on the details of both the intervention and its implementation. In particular, it appears that the key success factor is the extent to which careful thought goes into integrating effective pedagogical techniques with technology. Much more, and much more careful, research is needed (on both process and impacts) before committing resources to scaling up these programs - especially those involving expensive investments in hardware - with scarce public funds." Glewwe and Muralidharan 2016, Pg. 710. "Taken together, there is significant overlap in these recommendations: Computer-assisted learning or teacher-led interventions that individualize instruction can be highly effective. But pedagogical interventions or computing interventions generally are not inherently more effective than others; they have to be well implemented and affect students’ learning experience." Evans and Popova 2016, Pg. 260. "On the other hand, there is a great deal of excitement about video-based teaching by expert teachers, say from the Khan Academy, in teaching communities worldwide, and ICT [information and communication technology] is becoming increasingly sophisticated and cheaper by the day. The ICT-related studies in this review focus on computers; there is very little rigorous evidence on the potential for mobile phones or tablets, which may prove more practical in developing-country settings than computers. At the same time, it is likely that the supply of teachers who can effectively teach science or history at a relatively advanced level is not keeping pace with demand. Given all that, it seems very important to continue to explore the possibilities of using ICT in many alternative modalities, and not to get discouraged by the relatively negative experience thus far. In particular, integration of ICT materials into the overall pedagogy remains a challenge, as does ensuring that the students use the computers or other devices for the designated purpose rather than just playing with them. Other potentially interesting areas to explore within ICT include the use of virtual labs – can they replace a more hands on approach – and the use of e-readers (or for that matter libraries), cell phones, and texting to promote the pleasure of reading and non-curricular learning." Banerjee et al. 2013, Pg. 52. 

 114. "Evidence on the impact of monitoring on time in school is scarce and not encouraging, as seen in the first row of Table 10. Only two studies have considered this impact, those by Duflo et al. (2012b) and Banerjee et al. (2010). Both studies found insignificant effects of monitoring on student attendance… The evidence of the impact of monitoring on student learning is only somewhat more encouraging. The first row of Table 11 presents the results for four experimental (RCT) studies. In only one of the four studies was there a significantly positive impact of monitoring on students’ test scores. This is the study of Duflo et al. (2012b), who conducted a randomized evaluation of an intervention that monitored teacher attendance in informal schools in Rajasthan (India) using cameras with time-date stamps to record teacher and student attendance. The program not only monitored teachers, but also paid teacher salaries as a function of the number of valid days of attendance. They found that this program reduced teacher absence by half, but structural estimates of a model of labor supply suggest that the mechanism for this result was not the “monitoring” per se, but rather the incentives tied to the attendance. In contrast, no significant impact was found by Muralidharan and Sundararaman (2010), who experimentally studied the impact of a program that provided schools and teachers with low-stakes monitoring and feedback; they found that this program had no impact on either teacher attendance or test scores. These results suggest that while “monitoring” is an important tool in reducing teacher absence, “low-stakes” monitoring is unlikely to be very effective, and that it is “high-stakes” monitoring, that is monitoring with positive (negative) consequences for teacher presence (absence), which is more likely to be effective." Glewwe and Muralidharan 2016, Pg. 714. "A different way to improve monitoring of schools is to increase the amount of “bottom-up” monitoring through the community. The evidence here is less encouraging. Banerjee et al. (2010) conducted an experimental evaluation of the impact of a community mobilization program to improve school quality in rural areas of the Indian state of Uttar Pradesh; they found no impact of various programs to build community involvement in schools in that state on community participation, teacher effort, or learning outcomes." Glewwe and Muralidharan 2016, Pg. 714. "There is some positive evidence on the impact of community-based information campaigns (aimed at improving bottom-up monitoring) but the interventions have typically been quite intensive. Pandey et al. (2009) conducted an experimental evaluation of an information campaign to improve parental participation in village education committees (VEC’s) in three states in India and found positive impacts on both process measures as well as learning outcomes, but the estimated impacts on learning outcomes were generally statistically insignificant." Glewwe and Muralidharan 2016, Pg. 715. 

 115. "Another approach to improve monitoring and accountability of schools and teachers is to decentralize more management authority to schools and communities - an approach that is broadly referred to as “school-based management” (SBM). The theory of change associated with this approach is to empower communities to take charge of their schools and in particular to make teachers more accountable to them. Several reforms based on this approach have been attempted around the developing world, but the empirical evidence on its success is both limited and mixed. Five high-quality studies, three RCTs and two non-RCT studies, have examined the impact of SBM on children’s time in school. These five studies have produced 16 estimates, of which all but one are statistically insignificant, as seen in Table 10. Turning to student learning, seven high-quality studies have estimated the impact of SBM on students’ test scores, as seen Table 11, of which five are RCTs and two are non-RCT studies. Most of the 21 estimates from these seven studies are statistically insignificant, but three estimates from three different studies are significantly positive. We discuss these studies further below." Glewwe and Muralidharan 2016, Pg. 715.

 

 116. "Nevertheless, the demonstrated low levels of teacher effort in developing countries (manifested by high rates of absence) have led both policymakers and researchers to consider the possibility that introducing performance-linked pay for teachers may improve outcomes. Four high-quality studies have been conducted on this topic in recent years in developing countries. As seen in Table 10, one of these studies examined the impact of this type of education policy on students’ time in school. All four studies examined the impact of this type of intervention on test scores (see lines 4-5 in Table 11), most of which have found significantly positive impacts." Glewwe and Muralidharan 2016, Pg. 717. "There is also some evidence that teacher and school level incentives matter in the public system, but the evidence is not robust, and this issue can be very politically sensitive. Moreover, very little is known about incentive design. For example, there is some intriguing evidence from primary schools in Andhra Pradesh (India) that the recognition aspect of incentive payments matters as much as the actual cash amount (Muralidharan and Sundararaman 2011). Is this true in the post-primary context, where doing a good job perhaps requires more mastery of the material and relatively less physical energy and enthusiasm? There is also the question of who will provide the incentives: In the Muralidharan and Sundararaman study just mentioned, an NGO was responsible for distributing the incentive payments and teachers claimed that was important because they did not trust the government to honor its commitments. How do incentives get institutionalized? For example, can parents’ associations be involved in that process?" Banerjee et al. 2013, Pg. 51. 

 117. "While there is very little evidence on the impact of changing the employment contract structure of teachers while holding all else constant, as seen in Table 11 there is evidence from two high-quality studies on the impact of “contract teachers” on students’ test scores, although the specific ways in which such teachers differ from regular civil service teachers vary across contexts. In the first study, Duflo et al. (2015) present results from an experimental evaluation of a program in Kenya that provided a randomly selected set of schools with an extra contract teacher… As explained above (Section 4.2.3), Duflo et al. (2015) found that simply reducing class sizes had no significant impact on test scores. More relevant for teacher contract structure is that they found that students who had the reduced class sizes and were also taught by a contract teacher scored significantly higher (0.29σ, averaged across subjects) than those in control schools. Even more relevant is that they found that holding class size constant, students taught by contract teachers scored significantly higher than those taught by civil-service teachers even though the contract teachers are paid much lower salaries." Glewwe and Muralidharan 2016, Pg. 721. "The second high-quality study is that of Muralidharan and Sundararaman (2013a), who present experimental evidence from a program that provided an extra contract teacher to 100 randomly chosen government-run rural primary schools in the Indian state of Andhra Pradesh. At the end of 2 years, students in schools with an extra contract teacher performed significantly better than those in comparison schools by 0.16σ and 0.15σ in math and language tests, respectively… In fact, their point estimates typically suggest that the contract teachers are more effective than regular teachers, who are more qualified, better trained, and paid salaries five times higher than those of contract teachers." Glewwe and Muralidharan 2016, Pg. 721. 

 118. "First, the design of the details of any intervention matter enormously and should be more theoretically informed." Glewwe and Muralidharan 2016, Pg. 656. "Further, even when there are multiple studies of a similar intervention, there are almost always variations in the specific details of the intervention that make comparisons difficult. Take the case of teacher performance pay. While there are multiple high-quality studies on the subject, no two studies have the same formula for how teachers will be paid bonuses! Some of the design details that vary include individual versus group incentives, tournaments versus piece rates, linear versus nonlinear bonus formulae, formulae based on students reaching thresholds (such as the fraction who pass a test) versus those that reward improvements for all students. Similarly, interventions of technology-enabled learning vary from simply providing hardware to different forms of integration of technology into pedagogy, practice, assessment, and customizing of learning pathways." Glewwe and Muralidharan 2016, Pg. 728. 

 119. "The framework at the start of this section emphasizes that what happens in the schooling ages and later is likely to be conditioned on preschool education-related experience. The inputs into education obtained through schooling, for example, importantly include children and their pre-school levels of cognitive and physical development. Improving those inputs is likely to increase the productivity of subsequent education and thereby the incentives to invest in such education. Heckman (2006), for example, has suggested that for poor children the rates of return to improved preschool development may exceed significantly those for investments later in the life cycle." Behrman 2010, Pg. 4911. "Third, it is particularly important to try to understand why well-intentioned interventions may not have much of an impact on learning outcomes, and to aim such inquiry toward identifying binding constraints in the education system of interest. Fourth, interactions across components of an education system are likely to be of first order importance, which poses a challenge for most traditional research methods, which typically are not well suited to studying interactions." Glewwe and Muralidharan 2016, Pg. 656. "In practice, there are two main challenges when attempting to estimate the relationship in either Eq. (1) or Eq. (4). The first is that these equations represent the relationship between inputs and the total stock of human capital. Thus, estimating the production function in Eq. (1) would require the econometrician to have data on all prior inputs into human capital — including early childhood experiences and even in-utero conditions. This is an extremely challenging requirement and is unlikely to be feasible in almost all settings. Thus, the standard approach to estimating education production functions is to treat the lagged test score as a sufficient statistic for representing prior inputs into learning, and to use a value-added model to study the impact of changing contemporaneous inputs into education on test scores." Glewwe and Muralidharan 2016, Pg. 669. 

 120. "The effects we measure should be interpreted as conditional on the macro-economic context at the time, as emphasized by Rosenzweig and Udry (2016).... Overall, it seems plausible that the macro-economic conditions at the time the study cohort entered the labor market, and the government policy changes since the baseline, led both to lower overall labor-market performance for youth, and to lower treatment effects of education, than would have been present for a typical cohort, or would have been expected by participants at baseline." Duflo, Dupas, and Kremer 2017, Pg. 8.

 

 121. "Finally, while several individual high-quality studies have been produced in the past decade, more public goods and standards for measurement and reporting need to be created to make it easier for highly decentralized (and often opportunistically conducted) research studies to be compared across contexts. This will contribute to a more systematic understanding of the most cost-effective ways to improve education outcomes in developing countries." Glewwe and Muralidharan 2016, Pg. 656.

 

 122. "Second, while the standard education production function approach seeks to estimate population average parameters, heterogeneity across students is likely to be a first order issue. Thus, the optimal policy is likely to be different at different points in the student learning distribution." Glewwe and Muralidharan 2016, Pg. 656.

 

 123. As discussed in more detail above, we are not sure whether reductions in fertility and marriage rates are driven by income or educational effects of interventions studied and whether they are driven by 'incarceration effects' or more persistent changes to preferences, information, and/or the opportunity cost of having children. 

 

 124. See Row 94 on Tab "DDK (2017)"

 

 125. See Row 77 on Tab "Bettinger et al (2017)"

 

 126. The program's main effects were on pregnancy of girls under 16 and marriage of girls under 21. We make subjective value judgments to express these social outcomes in terms of dollars. See Rows 21-28 on Tab "DDK (2015)"

 

 127. In addition, we calculate the total cost of the policy and use the estimates of the effects of being randomly assigned to the treatment group (intent-to-treat) on the benefit side. Another approach would be to use the costs and benefits of the additional years of schooling only. 

 

 128. "We also observe that much of the variation in outcomes across educational interventions is captured within categories of interventions rather than across them. For example, saying that computer interventions are most effective may be less useful and less accurate than saying that computer-assisted learning programs that are tailored to each student’s level of knowledge, that are tied to the curriculum, and that provide teachers with training on how to integrate the technology into their instruction are most effective." Evans and Popova 2016, Pg. 244.

 

                          Contact Stay updated FAQ For Charities Site map Privacy Policy Jobs       Follow Us: Facebook Twitter RSS   Subscribe to email updates:  GiveWell, aka The Clear Fund (a tax-exempt 501(c)(3) public charity), was founded in 2007. We serve donors across the Globe; GiveWell's donors are based primarily in the United States, United Kingdom, Australia, Germany, and Canada. This work is licensed under a Creative Commons Attribution-Noncommercial-Share alike 3.0 United States License              try { clicky.init(78566); }catch(e){} 

     window.intercomSettings = { "app_id": "you5eu27" };   (function($) { $(document).ready(function() { // const tippy = require('tippy.js').default; tippy('.tippy', { content: '[data-tippy-content]', followCursor: "true", placement: "bottom", }); }); window.addEventListener('scroll', () => { tippy.hideAll() }) })(jQuery);