{ "__type": "IngestedDoc", "__tag": 4010, "_content": { "Notes": { "__type": "Section", "__tag": 4015, "children": [ { "__type": "Paragraph", "__tag": 4045, "children": [ { "__type": "Strong", "__tag": 4048, "children": [ { "__type": "Text", "__tag": 4046, "value": "Array API Standard Support" } ] } ] }, { "__type": "Paragraph", "__tag": 4045, "children": [ { "__type": "CrossRef", "__tag": 4002, "value": "monte_carlo_test", "reference": { "__type": "LocalRef", "__tag": 4022, "kind": "module", "path": "scipy.stats._resampling:monte_carlo_test" }, "kind": "module" }, { "__type": "Text", "__tag": 4046, "value": " has experimental support for Python Array API Standard compatible backends in addition to NumPy. Please consider testing these features by setting an environment variable " }, { "__type": "InlineCode", "__tag": 4051, "value": "SCIPY_ARRAY_API=1" }, { "__type": "Text", "__tag": 4046, "value": " and providing CuPy, PyTorch, JAX, or Dask arrays as array arguments. The following combinations of backend and device (or other capability) are supported." } ] }, { "__type": "Code", "__tag": 4050, "value": "==================== ==================== ====================\nLibrary CPU GPU\n==================== ==================== ====================\nNumPy ✅ n/a \nCuPy n/a ✅ \nPyTorch ✅ ✅ \nJAX ✅ ✅ \nDask ✅ n/a \n==================== ==================== ====================", "execution_status": null }, { "__type": "Blockquote", "__tag": 4059, "children": [ { "__type": "Paragraph", "__tag": 4045, "children": [ { "__type": "Text", "__tag": 4046, "value": "See " }, { "__type": "InlineRole", "__tag": 4003, "value": "dev-arrayapi", "domain": null, "role": "ref", "inventory": null }, { "__type": "Text", "__tag": 4046, "value": " for more information." } ] } ] } ], "title": [], "level": 0, "target": null }, "Warns": { "__type": "Section", "__tag": 4015, "children": [], "title": [], "level": 0, "target": null }, "Raises": { "__type": "Section", "__tag": 4015, "children": [], "title": [], "level": 0, "target": null }, "Yields": { "__type": "Section", "__tag": 4015, "children": [], "title": [], "level": 0, "target": null }, "Methods": { "__type": "Section", "__tag": 4015, "children": [], "title": [], "level": 0, "target": null }, "Returns": { "__type": "Section", "__tag": 4015, "children": [ { "__type": "Parameters", "__tag": 4026, "children": [ { "__type": "DocParam", "__tag": 4016, "name": "res", "annotation": "MonteCarloTestResult", "desc": [ { "__type": "Paragraph", "__tag": 4045, "children": [ { "__type": "Text", "__tag": 4046, "value": "An object with attributes:" } ] }, { "__type": "DefList", "__tag": 4033, "children": [ { "__type": "DefListItem", "__tag": 4037, "dt": { "__type": "Paragraph", "__tag": 4045, "children": [ { "__type": "Text", "__tag": 4046, "value": "statistic" } ] }, "dd": [ { "__type": "Paragraph", "__tag": 4045, "children": [ { "__type": "Text", "__tag": 4046, "value": "statistic" } ] } ] }, { "__type": "DefListItem", "__tag": 4037, "dt": { "__type": "Paragraph", "__tag": 4045, "children": [ { "__type": "Text", "__tag": 4046, "value": "pvalue" } ] }, "dd": [ { "__type": "Paragraph", "__tag": 4045, "children": [ { "__type": "Text", "__tag": 4046, "value": "pvalue" } ] } ] }, { "__type": "DefListItem", "__tag": 4037, "dt": { "__type": "Paragraph", "__tag": 4045, "children": [ { "__type": "Text", "__tag": 4046, "value": "null_distribution" } ] }, "dd": [ { "__type": "Paragraph", "__tag": 4045, "children": [ { "__type": "Text", "__tag": 4046, "value": "null_distribution" } ] } ] } ] } ] }, { "__type": "DocParam", "__tag": 4016, "name": "", "annotation": ".. warning::", "desc": [ { "__type": "Paragraph", "__tag": 4045, "children": [ { "__type": "Text", "__tag": 4046, "value": "The p-value is calculated by counting the elements of the null distribution that are as extreme or more extreme than the observed value of the statistic. Due to the use of finite precision arithmetic, some statistic functions return numerically distinct values when the theoretical values would be exactly equal. In some cases, this could lead to a large error in the calculated p-value. " }, { "__type": "CrossRef", "__tag": 4002, "value": "monte_carlo_test", "reference": { "__type": "LocalRef", "__tag": 4022, "kind": "module", "path": "scipy.stats._resampling:monte_carlo_test" }, "kind": "module" }, { "__type": "Text", "__tag": 4046, "value": " guards against this by considering elements in the null distribution that are \"close\" (within a relative tolerance of 100 times the floating point epsilon of inexact dtypes) to the observed value of the test statistic as equal to the observed value of the test statistic. However, the user is advised to inspect the null distribution to assess whether this method of comparison is appropriate, and if not, calculate the p-value manually." } ] } ] } ] } ], "title": [], "level": 0, "target": null }, "Summary": { "__type": "Section", "__tag": 4015, "children": [ { "__type": "Paragraph", "__tag": 4045, "children": [ { "__type": "Text", "__tag": 4046, "value": "Perform a Monte Carlo hypothesis test." } ] } ], "title": [], "level": 0, "target": null }, "Receives": { "__type": "Section", "__tag": 4015, "children": [], "title": [], "level": 0, "target": null }, "Warnings": { "__type": "Section", "__tag": 4015, "children": [], "title": [], "level": 0, "target": null }, "Attributes": { "__type": "Section", "__tag": 4015, "children": [], "title": [], "level": 0, "target": null }, "Parameters": { "__type": "Section", "__tag": 4015, "children": [ { "__type": "Parameters", "__tag": 4026, "children": [ { "__type": "DocParam", "__tag": 4016, "name": "data", "annotation": "array-like or sequence of array-like", "desc": [ { "__type": "Paragraph", "__tag": 4045, "children": [ { "__type": "Text", "__tag": 4046, "value": "An array or sequence of arrays of observations." } ] } ] }, { "__type": "DocParam", "__tag": 4016, "name": "rvs", "annotation": "callable or tuple of callables", "desc": [ { "__type": "Paragraph", "__tag": 4045, "children": [ { "__type": "Text", "__tag": 4046, "value": "A callable or sequence of callables that generates random variates under the null hypothesis. Each element of " }, { "__type": "ParamRef", "__tag": 4071, "name": "rvs" }, { "__type": "Text", "__tag": 4046, "value": " must be a callable that accepts keyword argument " }, { "__type": "InlineCode", "__tag": 4051, "value": "size" }, { "__type": "Text", "__tag": 4046, "value": " (e.g. " }, { "__type": "InlineCode", "__tag": 4051, "value": "rvs(size=(m, n))" }, { "__type": "Text", "__tag": 4046, "value": ") and returns an N-d array sample of that shape. If " }, { "__type": "ParamRef", "__tag": 4071, "name": "rvs" }, { "__type": "Text", "__tag": 4046, "value": " is a sequence, the number of callables in " }, { "__type": "ParamRef", "__tag": 4071, "name": "rvs" }, { "__type": "Text", "__tag": 4046, "value": " must match the number of samples in " }, { "__type": "ParamRef", "__tag": 4071, "name": "data" }, { "__type": "Text", "__tag": 4046, "value": ", i.e. " }, { "__type": "InlineCode", "__tag": 4051, "value": "len(rvs) == len(data)" }, { "__type": "Text", "__tag": 4046, "value": ". If " }, { "__type": "ParamRef", "__tag": 4071, "name": "rvs" }, { "__type": "Text", "__tag": 4046, "value": " is a single callable, " }, { "__type": "ParamRef", "__tag": 4071, "name": "data" }, { "__type": "Text", "__tag": 4046, "value": " is treated as a single sample." } ] } ] }, { "__type": "DocParam", "__tag": 4016, "name": "statistic", "annotation": "callable", "desc": [ { "__type": "Paragraph", "__tag": 4045, "children": [ { "__type": "Text", "__tag": 4046, "value": "Statistic for which the p-value of the hypothesis test is to be calculated. " }, { "__type": "ParamRef", "__tag": 4071, "name": "statistic" }, { "__type": "Text", "__tag": 4046, "value": " must be a callable that accepts a sample (e.g. " }, { "__type": "InlineCode", "__tag": 4051, "value": "statistic(sample)" }, { "__type": "Text", "__tag": 4046, "value": ") or " }, { "__type": "InlineCode", "__tag": 4051, "value": "len(rvs)" }, { "__type": "Text", "__tag": 4046, "value": " separate samples (e.g. " }, { "__type": "InlineCode", "__tag": 4051, "value": "statistic(samples1, sample2)" }, { "__type": "Text", "__tag": 4046, "value": " if " }, { "__type": "ParamRef", "__tag": 4071, "name": "rvs" }, { "__type": "Text", "__tag": 4046, "value": " contains two callables and " }, { "__type": "ParamRef", "__tag": 4071, "name": "data" }, { "__type": "Text", "__tag": 4046, "value": " contains two samples) and returns the resulting statistic. If " }, { "__type": "ParamRef", "__tag": 4071, "name": "vectorized" }, { "__type": "Text", "__tag": 4046, "value": " is set " }, { "__type": "InlineCode", "__tag": 4051, "value": "True" }, { "__type": "Text", "__tag": 4046, "value": ", " }, { "__type": "ParamRef", "__tag": 4071, "name": "statistic" }, { "__type": "Text", "__tag": 4046, "value": " must also accept a keyword argument " }, { "__type": "ParamRef", "__tag": 4071, "name": "axis" }, { "__type": "Text", "__tag": 4046, "value": " and be vectorized to compute the statistic along the provided " }, { "__type": "ParamRef", "__tag": 4071, "name": "axis" }, { "__type": "Text", "__tag": 4046, "value": " of the samples in " }, { "__type": "ParamRef", "__tag": 4071, "name": "data" }, { "__type": "Text", "__tag": 4046, "value": "." } ] } ] }, { "__type": "DocParam", "__tag": 4016, "name": "vectorized", "annotation": "bool, optional", "desc": [ { "__type": "Paragraph", "__tag": 4045, "children": [ { "__type": "Text", "__tag": 4046, "value": "If " }, { "__type": "ParamRef", "__tag": 4071, "name": "vectorized" }, { "__type": "Text", "__tag": 4046, "value": " is set " }, { "__type": "InlineCode", "__tag": 4051, "value": "False" }, { "__type": "Text", "__tag": 4046, "value": ", " }, { "__type": "ParamRef", "__tag": 4071, "name": "statistic" }, { "__type": "Text", "__tag": 4046, "value": " will not be passed keyword argument " }, { "__type": "ParamRef", "__tag": 4071, "name": "axis" }, { "__type": "Text", "__tag": 4046, "value": " and is expected to calculate the statistic only for 1D samples. If " }, { "__type": "InlineCode", "__tag": 4051, "value": "True" }, { "__type": "Text", "__tag": 4046, "value": ", " }, { "__type": "ParamRef", "__tag": 4071, "name": "statistic" }, { "__type": "Text", "__tag": 4046, "value": " will be passed keyword argument " }, { "__type": "ParamRef", "__tag": 4071, "name": "axis" }, { "__type": "Text", "__tag": 4046, "value": " and is expected to calculate the statistic along " }, { "__type": "ParamRef", "__tag": 4071, "name": "axis" }, { "__type": "Text", "__tag": 4046, "value": " when passed ND sample arrays. If " }, { "__type": "InlineCode", "__tag": 4051, "value": "None" }, { "__type": "Text", "__tag": 4046, "value": " (default), " }, { "__type": "ParamRef", "__tag": 4071, "name": "vectorized" }, { "__type": "Text", "__tag": 4046, "value": " will be set " }, { "__type": "InlineCode", "__tag": 4051, "value": "True" }, { "__type": "Text", "__tag": 4046, "value": " if " }, { "__type": "InlineCode", "__tag": 4051, "value": "axis" }, { "__type": "Text", "__tag": 4046, "value": " is a parameter of " }, { "__type": "ParamRef", "__tag": 4071, "name": "statistic" }, { "__type": "Text", "__tag": 4046, "value": ". Use of a vectorized statistic typically reduces computation time." } ] } ] }, { "__type": "DocParam", "__tag": 4016, "name": "n_resamples", "annotation": "int, default: 9999", "desc": [ { "__type": "Paragraph", "__tag": 4045, "children": [ { "__type": "Text", "__tag": 4046, "value": "Number of samples drawn from each of the callables of " }, { "__type": "ParamRef", "__tag": 4071, "name": "rvs" }, { "__type": "Text", "__tag": 4046, "value": ". Equivalently, the number statistic values under the null hypothesis used as the Monte Carlo null distribution." } ] } ] }, { "__type": "DocParam", "__tag": 4016, "name": "batch", "annotation": "int, optional", "desc": [ { "__type": "Paragraph", "__tag": 4045, "children": [ { "__type": "Text", "__tag": 4046, "value": "The number of Monte Carlo samples to process in each call to " }, { "__type": "ParamRef", "__tag": 4071, "name": "statistic" }, { "__type": "Text", "__tag": 4046, "value": ". Memory usage is O( " }, { "__type": "ParamRef", "__tag": 4071, "name": "batch" }, { "__type": "Text", "__tag": 4046, "value": " * " }, { "__type": "InlineCode", "__tag": 4051, "value": "sample.size[axis]" }, { "__type": "Text", "__tag": 4046, "value": " ). Default is " }, { "__type": "InlineCode", "__tag": 4051, "value": "None" }, { "__type": "Text", "__tag": 4046, "value": ", in which case " }, { "__type": "ParamRef", "__tag": 4071, "name": "batch" }, { "__type": "Text", "__tag": 4046, "value": " equals " }, { "__type": "ParamRef", "__tag": 4071, "name": "n_resamples" }, { "__type": "Text", "__tag": 4046, "value": "." } ] } ] }, { "__type": "DocParam", "__tag": 4016, "name": "alternative", "annotation": "{'two-sided', 'less', 'greater'}", "desc": [ { "__type": "Paragraph", "__tag": 4045, "children": [ { "__type": "Text", "__tag": 4046, "value": "The alternative hypothesis for which the p-value is calculated. For each alternative, the p-value is defined as follows." } ] }, { "__type": "BulletList", "__tag": 4053, "ordered": false, "start": 1, "children": [ { "__type": "ListItem", "__tag": 4054, "children": [ { "__type": "Paragraph", "__tag": 4045, "children": [ { "__type": "InlineCode", "__tag": 4051, "value": "'greater'" }, { "__type": "Text", "__tag": 4046, "value": "the percentage of the null distribution that is greater than or equal to the observed value of the test statistic." } ] } ] }, { "__type": "ListItem", "__tag": 4054, "children": [ { "__type": "Paragraph", "__tag": 4045, "children": [ { "__type": "InlineCode", "__tag": 4051, "value": "'less'" }, { "__type": "Text", "__tag": 4046, "value": "the percentage of the null distribution that is less than or equal to the observed value of the test statistic." } ] } ] }, { "__type": "ListItem", "__tag": 4054, "children": [ { "__type": "Paragraph", "__tag": 4045, "children": [ { "__type": "InlineCode", "__tag": 4051, "value": "'two-sided'" }, { "__type": "Text", "__tag": 4046, "value": "twice the smaller of the p-values above." } ] } ] } ] } ] }, { "__type": "DocParam", "__tag": 4016, "name": "axis", "annotation": "int, default: 0", "desc": [ { "__type": "Paragraph", "__tag": 4045, "children": [ { "__type": "Text", "__tag": 4046, "value": "The axis of " }, { "__type": "ParamRef", "__tag": 4071, "name": "data" }, { "__type": "Text", "__tag": 4046, "value": " (or each sample within " }, { "__type": "ParamRef", "__tag": 4071, "name": "data" }, { "__type": "Text", "__tag": 4046, "value": ") over which to calculate the statistic." } ] } ] } ] } ], "title": [], "level": 0, "target": null }, "Extended Summary": { "__type": "Section", "__tag": 4015, "children": [ { "__type": "Paragraph", "__tag": 4045, "children": [ { "__type": "ParamRef", "__tag": 4071, "name": "data" }, { "__type": "Text", "__tag": 4046, "value": " contains a sample or a sequence of one or more samples. " }, { "__type": "ParamRef", "__tag": 4071, "name": "rvs" }, { "__type": "Text", "__tag": 4046, "value": " specifies the distribution(s) of the sample(s) in " }, { "__type": "ParamRef", "__tag": 4071, "name": "data" }, { "__type": "Text", "__tag": 4046, "value": " under the null hypothesis. The value of " }, { "__type": "ParamRef", "__tag": 4071, "name": "statistic" }, { "__type": "Text", "__tag": 4046, "value": " for the given " }, { "__type": "ParamRef", "__tag": 4071, "name": "data" }, { "__type": "Text", "__tag": 4046, "value": " is compared against a Monte Carlo null distribution: the value of the statistic for each of " }, { "__type": "ParamRef", "__tag": 4071, "name": "n_resamples" }, { "__type": "Text", "__tag": 4046, "value": " sets of samples generated using " }, { "__type": "ParamRef", "__tag": 4071, "name": "rvs" }, { "__type": "Text", "__tag": 4046, "value": ". This gives the p-value, the probability of observing such an extreme value of the test statistic under the null hypothesis." } ] } ], "title": [], "level": 0, "target": null }, "Other Parameters": { "__type": "Section", "__tag": 4015, "children": [], "title": [], "level": 0, "target": null } }, "_ordered_sections": [ "Summary", "Extended Summary", "Parameters", "Attributes", "Methods", "Returns", "Yields", "Receives", "Other Parameters", "Raises", "Warns", "Warnings", "Notes" ], "item_file": "/scipy/stats/_resampling.py", "item_line": 792, "item_type": "function", "aliases": [ "scipy.stats.monte_carlo_test" ], "example_section_data": { "__type": "Section", "__tag": 4015, "children": [ { "__type": "Text", "__tag": 4046, "value": "Suppose we wish to test whether a small sample has been drawn from a normal\ndistribution. We decide that we will use the skew of the sample as a\ntest statistic, and we will consider a p-value of 0.05 to be statistically\nsignificant.\n\n" }, { "__type": "Code", "__tag": 4050, "value": "import numpy as np\nfrom scipy import stats\ndef statistic(x, axis):\n return stats.skew(x, axis)\n", "execution_status": "success" }, { "__type": "Text", "__tag": 4046, "value": "\nAfter collecting our data, we calculate the observed value of the test\nstatistic.\n\n" }, { "__type": "Code", "__tag": 4050, "value": "rng = np.random.default_rng()\nx = stats.skewnorm.rvs(a=1, size=50, random_state=rng)\n", "execution_status": "success" }, { "__type": "Code", "__tag": 4050, "value": "statistic(x, axis=0)\n", "execution_status": "failure" }, { "__type": "Text", "__tag": 4046, "value": "\nTo determine the probability of observing such an extreme value of the\nskewness by chance if the sample were drawn from the normal distribution,\nwe can perform a Monte Carlo hypothesis test. The test will draw many\nsamples at random from their normal distribution, calculate the skewness\nof each sample, and compare our original skewness against this\ndistribution to determine an approximate p-value.\n\n" }, { "__type": "Code", "__tag": 4050, "value": "from scipy.stats import monte_carlo_test\nrvs = lambda size: stats.norm.rvs(size=size, random_state=rng)\nres = monte_carlo_test(x, rvs, statistic, vectorized=True)\n", "execution_status": "success" }, { "__type": "Code", "__tag": 4050, "value": "print(res.statistic)\nprint(res.pvalue)\n", "execution_status": "failure" }, { "__type": "Text", "__tag": 4046, "value": "\nThe probability of obtaining a test statistic less than or equal to the\nobserved value under the null hypothesis is ~70%. This is greater than\nour chosen threshold of 5%, so we cannot consider this to be significant\nevidence against the null hypothesis.\n\nNote that this p-value essentially matches that of\n`scipy.stats.skewtest`, which relies on an asymptotic distribution of a\ntest statistic based on the sample skewness.\n\n" }, { "__type": "Code", "__tag": 4050, "value": "stats.skewtest(x).pvalue\n", "execution_status": "failure" }, { "__type": "Text", "__tag": 4046, "value": "\nThis asymptotic approximation is not valid for small sample sizes, but\n`monte_carlo_test` can be used with samples of any size.\n\n" }, { "__type": "Code", "__tag": 4050, "value": "x = stats.skewnorm.rvs(a=1, size=7, random_state=rng)\nres = monte_carlo_test(x, rvs, statistic, vectorized=True)\n", "execution_status": "success" }, { "__type": "Text", "__tag": 4046, "value": "\nThe Monte Carlo distribution of the test statistic is provided for\nfurther investigation.\n\n" }, { "__type": "Code", "__tag": 4050, "value": "import matplotlib.pyplot as plt\nfig, ax = plt.subplots()\n", "execution_status": "success" }, { "__type": "Code", "__tag": 4050, "value": "ax.hist(res.null_distribution, bins=50)\nax.set_title(\"Monte Carlo distribution of test statistic\")\nax.set_xlabel(\"Value of Statistic\")\nax.set_ylabel(\"Frequency\")\n", "execution_status": "failure" }, { "__type": "Code", "__tag": 4050, "value": "plt.show()\n", "execution_status": "success" }, { "__type": "Figure", "__tag": 4024, "value": { "__type": "RefInfo", "__tag": 4000, "module": "scipy", "version": "1.17.1", "kind": "assets", "path": "fig-fc4584a611c5ba72.png" } } ], "title": [], "level": 0, "target": null }, "see_also": [], "signature": { "__type": "SignatureNode", "__tag": 4029, "kind": "function", "parameters": [ { "__type": "SigParam", "__tag": 4030, "name": "data", "annotation": { "__type": "Empty", "__tag": 4031 }, "kind": "POSITIONAL_OR_KEYWORD", "default": { "__type": "Empty", "__tag": 4031 } }, { "__type": "SigParam", "__tag": 4030, "name": "rvs", "annotation": { "__type": "Empty", "__tag": 4031 }, "kind": "POSITIONAL_OR_KEYWORD", "default": { "__type": "Empty", "__tag": 4031 } }, { "__type": "SigParam", "__tag": 4030, "name": "statistic", "annotation": { "__type": "Empty", "__tag": 4031 }, "kind": "POSITIONAL_OR_KEYWORD", "default": { "__type": "Empty", "__tag": 4031 } }, { "__type": "SigParam", "__tag": 4030, "name": "vectorized", "annotation": { "__type": "Empty", "__tag": 4031 }, "kind": "KEYWORD_ONLY", "default": "None" }, { "__type": "SigParam", "__tag": 4030, "name": "n_resamples", "annotation": { "__type": "Empty", "__tag": 4031 }, "kind": "KEYWORD_ONLY", "default": "9999" }, { "__type": "SigParam", "__tag": 4030, "name": "batch", "annotation": { "__type": "Empty", "__tag": 4031 }, "kind": "KEYWORD_ONLY", "default": "None" }, { "__type": "SigParam", "__tag": 4030, "name": "alternative", "annotation": { "__type": "Empty", "__tag": 4031 }, "kind": "KEYWORD_ONLY", "default": "two-sided" }, { "__type": "SigParam", "__tag": 4030, "name": "axis", "annotation": { "__type": "Empty", "__tag": 4031 }, "kind": "KEYWORD_ONLY", "default": "0" } ], "return_annotation": { "__type": "Empty", "__tag": 4031 }, "target_name": "monte_carlo_test" }, "references": [ ".. [1] B. Phipson and G. K. Smyth. \"Permutation P-values Should Never Be", " Zero: Calculating Exact P-values When Permutations Are Randomly Drawn.\"", " Statistical Applications in Genetics and Molecular Biology 9.1 (2010)." ], "qa": "scipy.stats._resampling:monte_carlo_test", "arbitrary": [], "local_refs": [ "alternative", "axis", "batch", "data", "n_resamples", "res", "rvs", "statistic", "vectorized" ] }