{ "__type": "IngestedDoc", "__tag": 4010, "_content": {}, "_ordered_sections": [], "item_file": null, "item_line": null, "item_type": null, "aliases": [], "example_section_data": { "__type": "Section", "__tag": 4015, "children": [], "title": [], "level": 0, "target": null }, "see_also": [], "signature": null, "references": null, "qa": "reference:sparse.migration_to_sparray", "arbitrary": [ { "__type": "Section", "__tag": 4015, "children": [ { "__type": "Paragraph", "__tag": 4045, "children": [ { "__type": "Text", "__tag": 4046, "value": "This document provides guidance for converting code from sparse " }, { "__type": "Emphasis", "__tag": 4047, "children": [ { "__type": "Text", "__tag": 4046, "value": "matrices" } ] }, { "__type": "Text", "__tag": 4046, "value": " to sparse " }, { "__type": "Emphasis", "__tag": 4047, "children": [ { "__type": "Text", "__tag": 4046, "value": "arrays" } ] }, { "__type": "Text", "__tag": 4046, "value": " in " }, { "__type": "InlineCode", "__tag": 4051, "value": "scipy.sparse" }, { "__type": "Text", "__tag": 4046, "value": "." } ] }, { "__type": "Paragraph", "__tag": 4045, "children": [ { "__type": "Text", "__tag": 4046, "value": "The change from sparse matrices to sparse arrays mirrors conversion from " }, { "__type": "InlineCode", "__tag": 4051, "value": "np.matrix" }, { "__type": "Text", "__tag": 4046, "value": " to " }, { "__type": "InlineCode", "__tag": 4051, "value": "np.ndarray" }, { "__type": "Text", "__tag": 4046, "value": ". Essentially we must move from an all-2D matrix-multiplication-centric " }, { "__type": "InlineCode", "__tag": 4051, "value": "matrix" }, { "__type": "Text", "__tag": 4046, "value": " object to a 1D or 2D “array” object that supports the matrix multiplication operator and elementwise computation." } ] }, { "__type": "Paragraph", "__tag": 4045, "children": [ { "__type": "Text", "__tag": 4046, "value": "Notation: For this guide we denote the sparse array classes generally as " }, { "__type": "InlineCode", "__tag": 4051, "value": "sparray" }, { "__type": "Text", "__tag": 4046, "value": " and the sparse matrix classes " }, { "__type": "InlineCode", "__tag": 4051, "value": "spmatrix" }, { "__type": "Text", "__tag": 4046, "value": ". Dense numpy arrays are denoted " }, { "__type": "InlineCode", "__tag": 4051, "value": "np.ndarray" }, { "__type": "Text", "__tag": 4046, "value": " and dense matrix classes are " }, { "__type": "InlineCode", "__tag": 4051, "value": "np.matrix" }, { "__type": "Text", "__tag": 4046, "value": ". Supported sparse formats are denoted BSR, COO, CSC, CSR, DIA, DOK, LIL and all formats are supported by both sparray and spmatrix. The term " }, { "__type": "InlineCode", "__tag": 4051, "value": "sparse" }, { "__type": "Text", "__tag": 4046, "value": " refers to either " }, { "__type": "InlineCode", "__tag": 4051, "value": "sparray" }, { "__type": "Text", "__tag": 4046, "value": " or " }, { "__type": "InlineCode", "__tag": 4051, "value": "spmatrix" }, { "__type": "Text", "__tag": 4046, "value": ", while " }, { "__type": "InlineCode", "__tag": 4051, "value": "dense" }, { "__type": "Text", "__tag": 4046, "value": " refers to either " }, { "__type": "InlineCode", "__tag": 4051, "value": "np.ndarray" }, { "__type": "Text", "__tag": 4046, "value": " or " }, { "__type": "InlineCode", "__tag": 4051, "value": "np.matrix" }, { "__type": "Text", "__tag": 4046, "value": "." } ] } ], "title": [ { "__type": "Text", "__tag": 4046, "value": "Migration from spmatrix to sparray" } ], "level": 0, "target": "migration_to_sparray" }, { "__type": "Section", "__tag": 4015, "children": [ { "__type": "BulletList", "__tag": 4053, "ordered": false, "start": 1, "children": [ { "__type": "ListItem", "__tag": 4054, "children": [ { "__type": "Paragraph", "__tag": 4045, "children": [ { "__type": "Text", "__tag": 4046, "value": "The constructor names " }, { "__type": "InlineCode", "__tag": 4051, "value": "*_matrix" }, { "__type": "Text", "__tag": 4046, "value": ", e.g. " }, { "__type": "InlineCode", "__tag": 4051, "value": "csr_matrix" }, { "__type": "Text", "__tag": 4046, "value": ", are changed to " }, { "__type": "InlineCode", "__tag": 4051, "value": "*_array" }, { "__type": "Text", "__tag": 4046, "value": "." } ] } ] }, { "__type": "ListItem", "__tag": 4054, "children": [ { "__type": "Paragraph", "__tag": 4045, "children": [ { "__type": "Text", "__tag": 4046, "value": "spmatrix " }, { "__type": "InlineCode", "__tag": 4051, "value": "M" }, { "__type": "Text", "__tag": 4046, "value": " is always 2D (rows x columns) even e.g. " }, { "__type": "InlineCode", "__tag": 4051, "value": "M.min(axis=0)" }, { "__type": "Text", "__tag": 4046, "value": ". sparray " }, { "__type": "InlineCode", "__tag": 4051, "value": "A" }, { "__type": "Text", "__tag": 4046, "value": " can be 1D or 2D. Numpy scalars are returned for full (0D) reductions, i.e. " }, { "__type": "InlineCode", "__tag": 4051, "value": "M.min()" }, { "__type": "Text", "__tag": 4046, "value": "." } ] } ] }, { "__type": "ListItem", "__tag": 4054, "children": [ { "__type": "Paragraph", "__tag": 4045, "children": [ { "__type": "Text", "__tag": 4046, "value": "Iterating over a sparray gives 1D sparrays. Iterating spmatrix gives 2D row spmatrices" } ] } ] }, { "__type": "ListItem", "__tag": 4054, "children": [ { "__type": "Paragraph", "__tag": 4045, "children": [ { "__type": "Text", "__tag": 4046, "value": "Operators that change behavior are: " }, { "__type": "InlineCode", "__tag": 4051, "value": "*, @, *=, @=, **" } ] }, { "__type": "BulletList", "__tag": 4053, "ordered": false, "start": 1, "children": [ { "__type": "ListItem", "__tag": 4054, "children": [ { "__type": "Paragraph", "__tag": 4045, "children": [ { "__type": "Text", "__tag": 4046, "value": "Scalar multiplication, e.g. " }, { "__type": "InlineCode", "__tag": 4051, "value": "5 * A" }, { "__type": "Text", "__tag": 4046, "value": ", uses " }, { "__type": "InlineCode", "__tag": 4051, "value": "*" }, { "__type": "Text", "__tag": 4046, "value": ", and " }, { "__type": "InlineCode", "__tag": 4051, "value": "5 @ A" }, { "__type": "Text", "__tag": 4046, "value": " is not implemented." } ] } ] }, { "__type": "ListItem", "__tag": 4054, "children": [ { "__type": "Paragraph", "__tag": 4045, "children": [ { "__type": "Text", "__tag": 4046, "value": "sparrays use " }, { "__type": "InlineCode", "__tag": 4051, "value": "*" }, { "__type": "Text", "__tag": 4046, "value": " for elementwise multiplication and " }, { "__type": "InlineCode", "__tag": 4051, "value": "@" }, { "__type": "Text", "__tag": 4046, "value": " for matrix multiplication while spmatrices use either operator " }, { "__type": "InlineCode", "__tag": 4051, "value": "*" }, { "__type": "Text", "__tag": 4046, "value": " or " }, { "__type": "InlineCode", "__tag": 4051, "value": "@" }, { "__type": "Text", "__tag": 4046, "value": " for matrix multiplication. Either can use " }, { "__type": "InlineCode", "__tag": 4051, "value": "A.multiply(B)" }, { "__type": "Text", "__tag": 4046, "value": " for elementwise multiplication." } ] } ] }, { "__type": "ListItem", "__tag": 4054, "children": [ { "__type": "Paragraph", "__tag": 4045, "children": [ { "__type": "Text", "__tag": 4046, "value": "Scalar exponents, e.g. " }, { "__type": "InlineCode", "__tag": 4051, "value": "A**2" }, { "__type": "Text", "__tag": 4046, "value": ", use elementwise power for sparray and matrix power for spmatrix. Matrix power for sparrays uses " }, { "__type": "InlineCode", "__tag": 4051, "value": "scipy.sparse.linalg.matrix_power(A, n)" }, { "__type": "Text", "__tag": 4046, "value": "." } ] } ] } ] } ] }, { "__type": "ListItem", "__tag": 4054, "children": [ { "__type": "Paragraph", "__tag": 4045, "children": [ { "__type": "Text", "__tag": 4046, "value": "When index arrays are provided to the constructor functions, spmatrix selects a dtype based on dtype and values of the incoming arrays, while sparray only considers the dtype of the incoming arrays. For example, " }, { "__type": "InlineCode", "__tag": 4051, "value": "M=csr_matrix((data, indices, indptr))" }, { "__type": "Text", "__tag": 4046, "value": " results in " }, { "__type": "InlineCode", "__tag": 4051, "value": "int32" }, { "__type": "Text", "__tag": 4046, "value": " dtype for " }, { "__type": "InlineCode", "__tag": 4051, "value": "M.indices" }, { "__type": "Text", "__tag": 4046, "value": " so long as the values in " }, { "__type": "InlineCode", "__tag": 4051, "value": "indices" }, { "__type": "Text", "__tag": 4046, "value": " and " }, { "__type": "InlineCode", "__tag": 4051, "value": "indptr" }, { "__type": "Text", "__tag": 4046, "value": " are small, even if the " }, { "__type": "InlineCode", "__tag": 4051, "value": "dtype" }, { "__type": "Text", "__tag": 4046, "value": " of the incoming arrays are " }, { "__type": "InlineCode", "__tag": 4051, "value": "int64" }, { "__type": "Text", "__tag": 4046, "value": ". In contrast, " }, { "__type": "InlineCode", "__tag": 4051, "value": "A=csr_array((data, indices, indptr))" }, { "__type": "Text", "__tag": 4046, "value": " results in " }, { "__type": "InlineCode", "__tag": 4051, "value": "int64" }, { "__type": "Text", "__tag": 4046, "value": " dtype for " }, { "__type": "InlineCode", "__tag": 4051, "value": "A.indices" }, { "__type": "Text", "__tag": 4046, "value": " when the input arrays are " }, { "__type": "InlineCode", "__tag": 4051, "value": "int64" }, { "__type": "Text", "__tag": 4046, "value": ". This provides more predictable, often larger, index dtypes in sparrays and less casting to match dtypes." } ] } ] }, { "__type": "ListItem", "__tag": 4054, "children": [ { "__type": "Paragraph", "__tag": 4045, "children": [ { "__type": "Text", "__tag": 4046, "value": "Checking the sparse type and format:" } ] }, { "__type": "BulletList", "__tag": 4053, "ordered": false, "start": 1, "children": [ { "__type": "ListItem", "__tag": 4054, "children": [ { "__type": "Paragraph", "__tag": 4045, "children": [ { "__type": "InlineCode", "__tag": 4051, "value": "issparse(A)" }, { "__type": "Text", "__tag": 4046, "value": " returns " }, { "__type": "InlineCode", "__tag": 4051, "value": "True" }, { "__type": "Text", "__tag": 4046, "value": " for any sparse array/matrix." } ] } ] }, { "__type": "ListItem", "__tag": 4054, "children": [ { "__type": "Paragraph", "__tag": 4045, "children": [ { "__type": "InlineCode", "__tag": 4051, "value": "isspmatrix(M)" }, { "__type": "Text", "__tag": 4046, "value": " returns " }, { "__type": "InlineCode", "__tag": 4051, "value": "True" }, { "__type": "Text", "__tag": 4046, "value": " for any sparse matrix." } ] } ] }, { "__type": "ListItem", "__tag": 4054, "children": [ { "__type": "Paragraph", "__tag": 4045, "children": [ { "__type": "InlineCode", "__tag": 4051, "value": "isspmatrix_csr(M)" }, { "__type": "Text", "__tag": 4046, "value": " checks for a sparse matrix with specific format. It should be replaced with an array compatible version such as:" } ] } ] }, { "__type": "ListItem", "__tag": 4054, "children": [ { "__type": "Paragraph", "__tag": 4045, "children": [ { "__type": "InlineCode", "__tag": 4051, "value": "issparse(A) and A.format == 'csr'" }, { "__type": "Text", "__tag": 4046, "value": " which checks for a CSR sparse array/matrix." } ] } ] } ] } ] }, { "__type": "ListItem", "__tag": 4054, "children": [ { "__type": "Paragraph", "__tag": 4045, "children": [ { "__type": "Text", "__tag": 4046, "value": "Handling your software package API with sparse input/output:" } ] }, { "__type": "BulletList", "__tag": 4053, "ordered": false, "start": 1, "children": [ { "__type": "ListItem", "__tag": 4054, "children": [ { "__type": "Paragraph", "__tag": 4045, "children": [ { "__type": "Text", "__tag": 4046, "value": "Inputs are fairly easy to make work with either spmatrix or sparray. So long as you use " }, { "__type": "InlineCode", "__tag": 4051, "value": "A.multiply(B)" }, { "__type": "Text", "__tag": 4046, "value": " for elementwise and " }, { "__type": "InlineCode", "__tag": 4051, "value": "A @ B" }, { "__type": "Text", "__tag": 4046, "value": " for matrix multiplication, and you use " }, { "__type": "InlineCode", "__tag": 4051, "value": "sparse.linalg.matrix_power" }, { "__type": "Text", "__tag": 4046, "value": " for matrix power, you should be fine after you complete the \"first pass\" of the migration steps described in the next section. Your code will handle both types of inputs interchangeably." } ] } ] }, { "__type": "ListItem", "__tag": 4054, "children": [ { "__type": "Paragraph", "__tag": 4045, "children": [ { "__type": "Text", "__tag": 4046, "value": "Migrating sparse outputs from your functions requires a little more thought. Make a list of all your public functions that return spmatrix objects. Check whether you feel OK returning sparrays instead. That depends on your library and its users. If you want to allow these functions to continue to return spmatrix or sparray objects, you can often do that using a sparse input that also serves as a signal for what type of output should be returned. Design your function to return the type that was input. That approach can be extended to dense inputs. If the input is an np.matrix or a masked array with np.matrix as its " }, { "__type": "InlineCode", "__tag": 4051, "value": "._baseclass" }, { "__type": "Text", "__tag": 4046, "value": " attribute, then return spmatrix. Otherwise return an sparray. Without those inputs, two other approaches are to create a keyword argument to signal which to return, or create a new function (like we have done with, e.g. " }, { "__type": "InlineCode", "__tag": 4051, "value": "eye_array" }, { "__type": "Text", "__tag": 4046, "value": ") that has the same basic syntax, but returns sparray. Which method you choose should depend on your library and your users and your preferences." } ] } ] } ] } ] } ] } ], "title": [ { "__type": "Text", "__tag": 4046, "value": "Overview and big picture" } ], "level": 1, "target": null }, { "__type": "Section", "__tag": 4015, "children": [ { "__type": "BulletList", "__tag": 4053, "ordered": false, "start": 1, "children": [ { "__type": "ListItem", "__tag": 4054, "children": [ { "__type": "Paragraph", "__tag": 4045, "children": [ { "__type": "Text", "__tag": 4046, "value": "First pass (leaving spmatrix in the code):" } ] }, { "__type": "BulletList", "__tag": 4053, "ordered": false, "start": 1, "children": [ { "__type": "ListItem", "__tag": 4054, "children": [ { "__type": "Paragraph", "__tag": 4045, "children": [ { "__type": "Text", "__tag": 4046, "value": "In your spmatrix code, change " }, { "__type": "InlineCode", "__tag": 4051, "value": "*" }, { "__type": "Text", "__tag": 4046, "value": " to " }, { "__type": "InlineCode", "__tag": 4051, "value": "@" }, { "__type": "Text", "__tag": 4046, "value": " for matrix multiplication. Note that scalar multiplication with sparse should use " }, { "__type": "InlineCode", "__tag": 4051, "value": "*" }, { "__type": "Text", "__tag": 4046, "value": ". (See helper-code " }, { "__type": "CrossRef", "__tag": 4002, "value": "sparse-migration-star-vs-at", "reference": { "__type": "LocalRef", "__tag": 4022, "kind": "docs", "path": "reference:sparse.migration_to_sparray" }, "kind": "exists" }, { "__type": "Text", "__tag": 4046, "value": " below)" } ] } ] }, { "__type": "ListItem", "__tag": 4054, "children": [ { "__type": "Paragraph", "__tag": 4045, "children": [ { "__type": "Text", "__tag": 4046, "value": "Matrix powers, e.g. " }, { "__type": "InlineCode", "__tag": 4051, "value": "M**3" }, { "__type": "Text", "__tag": 4046, "value": ", should be converted to " }, { "__type": "InlineCode", "__tag": 4051, "value": "scipy.sparse.linalg.matrix_power(A, 3)" } ] } ] }, { "__type": "ListItem", "__tag": 4054, "children": [ { "__type": "Paragraph", "__tag": 4045, "children": [ { "__type": "Text", "__tag": 4046, "value": "Implement alternatives to unsupported functions/methods like " }, { "__type": "InlineCode", "__tag": 4051, "value": "A.getnnz()" }, { "__type": "Text", "__tag": 4046, "value": " -> " }, { "__type": "InlineCode", "__tag": 4051, "value": "A.nnz" }, { "__type": "Text", "__tag": 4046, "value": " (see " }, { "__type": "CrossRef", "__tag": 4002, "value": "sparse-migration-removed-methods", "reference": { "__type": "LocalRef", "__tag": 4022, "kind": "docs", "path": "reference:sparse.migration_to_sparray" }, "kind": "exists" }, { "__type": "Text", "__tag": 4046, "value": " below)." } ] } ] }, { "__type": "ListItem", "__tag": 4054, "children": [ { "__type": "Paragraph", "__tag": 4045, "children": [ { "__type": "Text", "__tag": 4046, "value": "Change any logic regarding " }, { "__type": "InlineCode", "__tag": 4051, "value": "issparse()" }, { "__type": "Text", "__tag": 4046, "value": " and " }, { "__type": "InlineCode", "__tag": 4051, "value": "isspmatrix()" }, { "__type": "Text", "__tag": 4046, "value": " as needed. Usually, this means replacing " }, { "__type": "InlineCode", "__tag": 4051, "value": "isspmatrix" }, { "__type": "Text", "__tag": 4046, "value": " with " }, { "__type": "InlineCode", "__tag": 4051, "value": "issparse" }, { "__type": "Text", "__tag": 4046, "value": ", and " }, { "__type": "InlineCode", "__tag": 4051, "value": "isspmatrix_csr(G)" }, { "__type": "Text", "__tag": 4046, "value": " with " }, { "__type": "InlineCode", "__tag": 4051, "value": "issparse(G) and G.format == \"csr\"" }, { "__type": "Text", "__tag": 4046, "value": ". Moreover " }, { "__type": "InlineCode", "__tag": 4051, "value": "isspmatrix_csr(G) or isspmatrix_csc(G)" }, { "__type": "Text", "__tag": 4046, "value": " becomes " }, { "__type": "InlineCode", "__tag": 4051, "value": "issparse(G) and G.format in ['csr', 'csc']" }, { "__type": "Text", "__tag": 4046, "value": ". The git search idiom " }, { "__type": "InlineCode", "__tag": 4051, "value": "git grep 'isspm[a-z_]*('" }, { "__type": "Text", "__tag": 4046, "value": " can help find these." } ] } ] }, { "__type": "ListItem", "__tag": 4054, "children": [ { "__type": "Paragraph", "__tag": 4045, "children": [ { "__type": "Text", "__tag": 4046, "value": "Convert all " }, { "__type": "InlineCode", "__tag": 4051, "value": "spdiags" }, { "__type": "Text", "__tag": 4046, "value": " calls to " }, { "__type": "InlineCode", "__tag": 4051, "value": "dia_matrix" }, { "__type": "Text", "__tag": 4046, "value": ". See docs in " }, { "__type": "InlineRole", "__tag": 4003, "value": "spdiags", "domain": null, "role": "func", "inventory": null }, { "__type": "Text", "__tag": 4046, "value": ". A search for " }, { "__type": "InlineCode", "__tag": 4051, "value": "spdiags" }, { "__type": "Text", "__tag": 4046, "value": " is all you need here." } ] } ] }, { "__type": "ListItem", "__tag": 4054, "children": [ { "__type": "Paragraph", "__tag": 4045, "children": [ { "__type": "Text", "__tag": 4046, "value": "Run all your tests on the resulting code. You are still using spmatrix, not sparray. But your code and tests are prepared for the change and you should be able to take sparrays as input to your code and have them mostly \"just work\"." } ] } ] } ] } ] }, { "__type": "ListItem", "__tag": 4054, "children": [ { "__type": "Paragraph", "__tag": 4045, "children": [ { "__type": "Text", "__tag": 4046, "value": "Second pass (switching to sparray):" } ] }, { "__type": "BulletList", "__tag": 4053, "ordered": false, "start": 1, "children": [ { "__type": "ListItem", "__tag": 4054, "children": [ { "__type": "Paragraph", "__tag": 4045, "children": [ { "__type": "Text", "__tag": 4046, "value": "Convert construction functions like " }, { "__type": "InlineCode", "__tag": 4051, "value": "diags" }, { "__type": "Text", "__tag": 4046, "value": " and " }, { "__type": "InlineCode", "__tag": 4051, "value": "triu" }, { "__type": "Text", "__tag": 4046, "value": " to the array version (see " }, { "__type": "CrossRef", "__tag": 4002, "value": "sparse-migration-construction", "reference": { "__type": "LocalRef", "__tag": 4022, "kind": "docs", "path": "reference:sparse.migration_to_sparray" }, "kind": "exists" }, { "__type": "Text", "__tag": 4046, "value": " below)." } ] } ] }, { "__type": "ListItem", "__tag": 4054, "children": [ { "__type": "Paragraph", "__tag": 4045, "children": [ { "__type": "Text", "__tag": 4046, "value": "Rename all " }, { "__type": "InlineCode", "__tag": 4051, "value": "*_matrix" }, { "__type": "Text", "__tag": 4046, "value": " constructor calls to " }, { "__type": "InlineCode", "__tag": 4051, "value": "*_array" }, { "__type": "Text", "__tag": 4046, "value": "." } ] } ] }, { "__type": "ListItem", "__tag": 4054, "children": [ { "__type": "Paragraph", "__tag": 4045, "children": [ { "__type": "Text", "__tag": 4046, "value": "Check all functions/methods for which migration causes 1D return values. These are mostly indexing and the reduction functions (see " }, { "__type": "CrossRef", "__tag": 4002, "value": "sparse-migration-shapes-reductions", "reference": { "__type": "LocalRef", "__tag": 4022, "kind": "docs", "path": "reference:sparse.migration_to_sparray" }, "kind": "exists" }, { "__type": "Text", "__tag": 4046, "value": " below)." } ] } ] }, { "__type": "ListItem", "__tag": 4054, "children": [ { "__type": "Paragraph", "__tag": 4045, "children": [ { "__type": "Text", "__tag": 4046, "value": "Check all places where you iterate over spmatrices and change them to account for the sparrays yielding 1D sparrays rather than 2D spmatrices." } ] } ] }, { "__type": "ListItem", "__tag": 4054, "children": [ { "__type": "Paragraph", "__tag": 4045, "children": [ { "__type": "Text", "__tag": 4046, "value": "Find and change places where your code makes use of " }, { "__type": "InlineCode", "__tag": 4051, "value": "np.matrix" }, { "__type": "Text", "__tag": 4046, "value": " features. Convert those to " }, { "__type": "InlineCode", "__tag": 4051, "value": "np.ndarray" }, { "__type": "Text", "__tag": 4046, "value": " features." } ] } ] }, { "__type": "ListItem", "__tag": 4054, "children": [ { "__type": "Paragraph", "__tag": 4045, "children": [ { "__type": "Text", "__tag": 4046, "value": "If your code reads sparse from files with " }, { "__type": "InlineCode", "__tag": 4051, "value": "mmread" }, { "__type": "Text", "__tag": 4046, "value": ", " }, { "__type": "InlineCode", "__tag": 4051, "value": "hb_read" }, { "__type": "Text", "__tag": 4046, "value": " or " }, { "__type": "InlineCode", "__tag": 4051, "value": "loadmat" }, { "__type": "Text", "__tag": 4046, "value": ", use the new keyword argument " }, { "__type": "InlineCode", "__tag": 4051, "value": "spmatrix=False" }, { "__type": "Text", "__tag": 4046, "value": " in those functions to read to sparray." } ] } ] }, { "__type": "ListItem", "__tag": 4054, "children": [ { "__type": "Paragraph", "__tag": 4045, "children": [ { "__type": "Text", "__tag": 4046, "value": "If you use sparse libraries that only accept " }, { "__type": "InlineCode", "__tag": 4051, "value": "int32" }, { "__type": "Text", "__tag": 4046, "value": " index arrays for sparse representations, we suggest using just-in-time conversion. Convert to " }, { "__type": "InlineCode", "__tag": 4051, "value": "int32" }, { "__type": "Text", "__tag": 4046, "value": " just before you call the code that requires " }, { "__type": "InlineCode", "__tag": 4051, "value": "int32" }, { "__type": "Text", "__tag": 4046, "value": "." } ] } ] }, { "__type": "ListItem", "__tag": 4054, "children": [ { "__type": "Paragraph", "__tag": 4045, "children": [ { "__type": "InlineCode", "__tag": 4051, "value": "sparray" }, { "__type": "Text", "__tag": 4046, "value": " selects index dtype based on the dtype of the input array instead of the values in the array. So if you want your index arrays to be " }, { "__type": "InlineCode", "__tag": 4051, "value": "int32" }, { "__type": "Text", "__tag": 4046, "value": ", you will need to ensure an " }, { "__type": "InlineCode", "__tag": 4051, "value": "int32" }, { "__type": "Text", "__tag": 4046, "value": " dtype for each index array like " }, { "__type": "InlineCode", "__tag": 4051, "value": "indptr" }, { "__type": "Text", "__tag": 4046, "value": " that you pass to " }, { "__type": "InlineCode", "__tag": 4051, "value": "csr_array" }, { "__type": "Text", "__tag": 4046, "value": ". With " }, { "__type": "InlineCode", "__tag": 4051, "value": "spmatrix" }, { "__type": "Text", "__tag": 4046, "value": " it is tempting to use the default int64 dtype for " }, { "__type": "InlineCode", "__tag": 4051, "value": "numpy" }, { "__type": "Text", "__tag": 4046, "value": " arrays and rely on the sparse constructor to downcast if the values were small. But this downcasting leads to extra recasting when working with other matrices, slices or arithmetic expressions. For " }, { "__type": "InlineCode", "__tag": 4051, "value": "sparray" }, { "__type": "Text", "__tag": 4046, "value": " you can still rely on the constructors to choose dtypes. But you are also given the power to choose your index dtype via the dtype of the incoming index arrays rather than their values. So, if you want " }, { "__type": "InlineCode", "__tag": 4051, "value": "int32" }, { "__type": "Text", "__tag": 4046, "value": ", set the dtype, e.g. " }, { "__type": "InlineCode", "__tag": 4051, "value": "indices = np.array([1,3,6], dtype=np.int32)" }, { "__type": "Text", "__tag": 4046, "value": " or " }, { "__type": "InlineCode", "__tag": 4051, "value": "indptr = np.arange(9, dtype=np.int32)" }, { "__type": "Text", "__tag": 4046, "value": ", when creating the index arrays. See " }, { "__type": "CrossRef", "__tag": 4002, "value": "sparse-migration-index-array-dtypes", "reference": { "__type": "LocalRef", "__tag": 4022, "kind": "docs", "path": "reference:sparse.migration_to_sparray" }, "kind": "exists" }, { "__type": "Text", "__tag": 4046, "value": " below for more info. In many settings, the index array dtype isn't crucial and you can just let the constructors choose the dtype for both sparray and spmatrix." } ] } ] }, { "__type": "ListItem", "__tag": 4054, "children": [ { "__type": "Paragraph", "__tag": 4045, "children": [ { "__type": "Text", "__tag": 4046, "value": "Test your code. And " }, { "__type": "Strong", "__tag": 4048, "children": [ { "__type": "Text", "__tag": 4046, "value": "read" } ] }, { "__type": "Text", "__tag": 4046, "value": " your code. You have migrated to sparray." } ] } ] } ] } ] } ] } ], "title": [ { "__type": "Text", "__tag": 4046, "value": "Recommended steps for migration" } ], "level": 1, "target": null }, { "__type": "Section", "__tag": 4015, "children": [ { "__type": "Paragraph", "__tag": 4045, "children": [ { "__type": "Text", "__tag": 4046, "value": "These four functions are new and only handle sparrays: " }, { "__type": "CrossRef", "__tag": 4002, "value": "block_array", "reference": { "__type": "RefInfo", "__tag": 4000, "module": "scipy", "version": "*", "kind": "api", "path": "scipy.sparse._construct:block_array" }, "kind": "module" }, { "__type": "Text", "__tag": 4046, "value": ", " }, { "__type": "CrossRef", "__tag": 4002, "value": "diags_array", "reference": { "__type": "RefInfo", "__tag": 4000, "module": "scipy", "version": "*", "kind": "api", "path": "scipy.sparse._construct:diags_array" }, "kind": "module" }, { "__type": "Text", "__tag": 4046, "value": ", " }, { "__type": "CrossRef", "__tag": 4002, "value": "eye_array", "reference": { "__type": "RefInfo", "__tag": 4000, "module": "scipy", "version": "*", "kind": "api", "path": "scipy.sparse._construct:eye_array" }, "kind": "module" }, { "__type": "Text", "__tag": 4046, "value": ", and " }, { "__type": "CrossRef", "__tag": 4002, "value": "random_array", "reference": { "__type": "RefInfo", "__tag": 4000, "module": "scipy", "version": "*", "kind": "api", "path": "scipy.sparse._construct:random_array" }, "kind": "module" }, { "__type": "Text", "__tag": 4046, "value": ". Their signatures are " } ] }, { "__type": "Code", "__tag": 4050, "value": "def block_array(blocks, format=None, dtype=None):\ndef diags_array(diagonals, /, *, offsets=0, shape=None, format=None, dtype=None):\ndef eye_array(m, n=None, *, k=0, dtype=float, format=None):\ndef random_array(shape, density=0.01, format='coo', dtype=None, rng=None, data_sampler=None):", "execution_status": null }, { "__type": "Paragraph", "__tag": 4045, "children": [ { "__type": "Text", "__tag": 4046, "value": "The " }, { "__type": "InlineCode", "__tag": 4051, "value": "random_array" }, { "__type": "Text", "__tag": 4046, "value": " function has a " }, { "__type": "InlineCode", "__tag": 4051, "value": "shape" }, { "__type": "Text", "__tag": 4046, "value": " (2-tuple) arg rather than two integers. And the " }, { "__type": "InlineCode", "__tag": 4051, "value": "rng" }, { "__type": "Text", "__tag": 4046, "value": " arg defaults to NumPy's new " }, { "__type": "InlineCode", "__tag": 4051, "value": "default_rng()" }, { "__type": "Text", "__tag": 4046, "value": ". This differs from the spmatrix " }, { "__type": "InlineCode", "__tag": 4051, "value": "rand" }, { "__type": "Text", "__tag": 4046, "value": " and " }, { "__type": "InlineCode", "__tag": 4051, "value": "random" }, { "__type": "Text", "__tag": 4046, "value": " which default to the global RandomState instance. If you don't care much about these things, leaving it as the default should work fine. If you care about seeding your random numbers, you should probably add a " }, { "__type": "InlineCode", "__tag": 4051, "value": "rng=..." }, { "__type": "Text", "__tag": 4046, "value": " keyword argument to this call when you switch functions. In summary, to migrate to " }, { "__type": "InlineCode", "__tag": 4051, "value": "random_array" }, { "__type": "Text", "__tag": 4046, "value": " change the function name, switch the shape argument to a single tuple argument, leave any other parameters as before, and think about what sort of " }, { "__type": "InlineCode", "__tag": 4051, "value": "rng=" }, { "__type": "Text", "__tag": 4046, "value": " argument should be used, if any." } ] }, { "__type": "Paragraph", "__tag": 4045, "children": [ { "__type": "Text", "__tag": 4046, "value": "The " }, { "__type": "InlineRole", "__tag": 4003, "value": "diags_array", "domain": null, "role": null, "inventory": null }, { "__type": "Text", "__tag": 4046, "value": " function uses keyword-only rules for arguments. So you have to type the " }, { "__type": "InlineRole", "__tag": 4003, "value": "offsets=", "domain": null, "role": null, "inventory": null }, { "__type": "Text", "__tag": 4046, "value": " in front of the offsets arguments. That seems like a pain during migration from using " }, { "__type": "InlineRole", "__tag": 4003, "value": "diags", "domain": null, "role": null, "inventory": null }, { "__type": "Text", "__tag": 4046, "value": ", but it helps avoid confusion and eases reading. A single shape parameter replaces two integers for this migration as well." } ] } ], "title": [ { "__type": "Text", "__tag": 4046, "value": "Details: construction functions" } ], "level": 1, "target": "sparse-migration-construction" }, { "__type": "Section", "__tag": 4015, "children": [ { "__type": "Paragraph", "__tag": 4045, "children": [ { "__type": "Text", "__tag": 4046, "value": "These functions return sparray or spmatrix, depending on the input types they receive: " }, { "__type": "CrossRef", "__tag": 4002, "value": "kron", "reference": { "__type": "RefInfo", "__tag": 4000, "module": "scipy", "version": "*", "kind": "api", "path": "scipy.sparse._construct:kron" }, "kind": "module" }, { "__type": "Text", "__tag": 4046, "value": ", " }, { "__type": "CrossRef", "__tag": 4002, "value": "kronsum", "reference": { "__type": "RefInfo", "__tag": 4000, "module": "scipy", "version": "*", "kind": "api", "path": "scipy.sparse._construct:kronsum" }, "kind": "module" }, { "__type": "Text", "__tag": 4046, "value": ", " }, { "__type": "CrossRef", "__tag": 4002, "value": "hstack", "reference": { "__type": "RefInfo", "__tag": 4000, "module": "scipy", "version": "*", "kind": "api", "path": "scipy.sparse._construct:hstack" }, "kind": "module" }, { "__type": "Text", "__tag": 4046, "value": ", " }, { "__type": "CrossRef", "__tag": 4002, "value": "vstack", "reference": { "__type": "RefInfo", "__tag": 4000, "module": "scipy", "version": "*", "kind": "api", "path": "scipy.sparse._construct:vstack" }, "kind": "module" }, { "__type": "Text", "__tag": 4046, "value": ", " }, { "__type": "CrossRef", "__tag": 4002, "value": "block_diag", "reference": { "__type": "RefInfo", "__tag": 4000, "module": "scipy", "version": "*", "kind": "api", "path": "scipy.sparse._construct:block_diag" }, "kind": "module" }, { "__type": "Text", "__tag": 4046, "value": ", " }, { "__type": "CrossRef", "__tag": 4002, "value": "tril", "reference": { "__type": "RefInfo", "__tag": 4000, "module": "scipy", "version": "*", "kind": "api", "path": "scipy.sparse._extract:tril" }, "kind": "module" }, { "__type": "Text", "__tag": 4046, "value": ", and " }, { "__type": "CrossRef", "__tag": 4002, "value": "triu", "reference": { "__type": "RefInfo", "__tag": 4000, "module": "scipy", "version": "*", "kind": "api", "path": "scipy.sparse._extract:triu" }, "kind": "module" }, { "__type": "Text", "__tag": 4046, "value": ". Their signatures are " } ] }, { "__type": "Code", "__tag": 4050, "value": "def kron(A, B, format=None):\ndef kronsum(A, B, format=None):\ndef hstack(blocks, format=None, dtype=None):\ndef vstack(blocks, format=None, dtype=None):\ndef block_diag(mats, format=None, dtype=None):\ndef tril(A, k=0, format=None):\ndef triu(A, k=0, format=None):", "execution_status": null }, { "__type": "Paragraph", "__tag": 4045, "children": [ { "__type": "Text", "__tag": 4046, "value": "Use of these functions should be examined and inputs adjusted to ensure return values are sparrays. And in turn the outputs should be treated as sparrays. To return sparrays, at least one input must be an sparray. If you use list-of-lists or numpy arrays as input you should convert one of them to a sparse array to get sparse arrays out." } ] } ], "title": [ { "__type": "Text", "__tag": 4046, "value": "Existing functions that need careful migration" } ], "level": 2, "target": null }, { "__type": "Section", "__tag": 4015, "children": [ { "__type": "Blockquote", "__tag": 4059, "children": [ { "__type": "Code", "__tag": 4050, "value": "========= ============= ==================================\nFunction New function Comments\n========= ============= ==================================\neye eye_array\nidentity eye_array\ndiags diags_array keyword-only input\nspdiags dia_array shape as 2-tuple\nbmat block\nrand random_array shape as 2-tuple and default_rng\nrandom random_array shape as 2-tuple and default_rng\n========= ============= ==================================", "execution_status": null } ] } ], "title": [ { "__type": "Text", "__tag": 4046, "value": "Functions that changed names for the migration" } ], "level": 2, "target": null }, { "__type": "Section", "__tag": 4015, "children": [ { "__type": "BulletList", "__tag": 4053, "ordered": false, "start": 1, "children": [ { "__type": "ListItem", "__tag": 4054, "children": [ { "__type": "Paragraph", "__tag": 4045, "children": [ { "__type": "Text", "__tag": 4046, "value": "Construction using 1d-list of values:" } ] }, { "__type": "BulletList", "__tag": 4053, "ordered": false, "start": 1, "children": [ { "__type": "ListItem", "__tag": 4054, "children": [ { "__type": "Paragraph", "__tag": 4045, "children": [ { "__type": "InlineCode", "__tag": 4051, "value": "csr_array([1, 2, 3]).shape == (3,)" }, { "__type": "Text", "__tag": 4046, "value": " 1D input makes a 1D array." } ] } ] }, { "__type": "ListItem", "__tag": 4054, "children": [ { "__type": "Paragraph", "__tag": 4045, "children": [ { "__type": "InlineCode", "__tag": 4051, "value": "csr_matrix([1, 2, 3]).shape == (1, 3)" }, { "__type": "Text", "__tag": 4046, "value": " 1D input makes a 2D matrix." } ] } ] } ] } ] }, { "__type": "ListItem", "__tag": 4054, "children": [ { "__type": "Paragraph", "__tag": 4045, "children": [ { "__type": "Text", "__tag": 4046, "value": "Indexing and iteration:" } ] }, { "__type": "BulletList", "__tag": 4053, "ordered": false, "start": 1, "children": [ { "__type": "ListItem", "__tag": 4054, "children": [ { "__type": "Paragraph", "__tag": 4045, "children": [ { "__type": "Text", "__tag": 4046, "value": "Indexing of sparray allows 1D objects which can be made 2D using " }, { "__type": "InlineCode", "__tag": 4051, "value": "np.newaxis" }, { "__type": "Text", "__tag": 4046, "value": " or " }, { "__type": "InlineCode", "__tag": 4051, "value": "None" }, { "__type": "Text", "__tag": 4046, "value": ". E.g., " }, { "__type": "InlineCode", "__tag": 4051, "value": "A[3, None, :]" }, { "__type": "Text", "__tag": 4046, "value": " gives a 2D row. Indexing of 2D sparray with implicit (not given) column index gives a 1D result, e.g. " }, { "__type": "InlineCode", "__tag": 4051, "value": "A[3]" }, { "__type": "Text", "__tag": 4046, "value": " (note: best not to do this - write it as " }, { "__type": "InlineCode", "__tag": 4051, "value": "A[3, :]" }, { "__type": "Text", "__tag": 4046, "value": " instead). If you need a 2D result, use " }, { "__type": "InlineCode", "__tag": 4051, "value": "np.newaxis" }, { "__type": "Text", "__tag": 4046, "value": ", or " }, { "__type": "InlineCode", "__tag": 4051, "value": "None" }, { "__type": "Text", "__tag": 4046, "value": " in your index, or wrap the integer index as a list for which fancy indexing gives 2D, e.g. " }, { "__type": "InlineCode", "__tag": 4051, "value": "A[[3], :]" }, { "__type": "Text", "__tag": 4046, "value": "." } ] } ] }, { "__type": "ListItem", "__tag": 4054, "children": [ { "__type": "Paragraph", "__tag": 4045, "children": [ { "__type": "Text", "__tag": 4046, "value": "Iteration over sparse object: " }, { "__type": "InlineCode", "__tag": 4051, "value": "next(M)" }, { "__type": "Text", "__tag": 4046, "value": " yields a sparse 2D row matrix, " }, { "__type": "InlineCode", "__tag": 4051, "value": "next(A)" }, { "__type": "Text", "__tag": 4046, "value": " yields a sparse 1D array." } ] } ] } ] } ] }, { "__type": "ListItem", "__tag": 4054, "children": [ { "__type": "Paragraph", "__tag": 4045, "children": [ { "__type": "Text", "__tag": 4046, "value": "Reduction operations along an axis reduce the shape:" } ] }, { "__type": "BulletList", "__tag": 4053, "ordered": false, "start": 1, "children": [ { "__type": "ListItem", "__tag": 4054, "children": [ { "__type": "Paragraph", "__tag": 4045, "children": [ { "__type": "InlineCode", "__tag": 4051, "value": "M.min(axis=1)" }, { "__type": "Text", "__tag": 4046, "value": " returns a 2D row matrix of the min along axis 1." } ] } ] }, { "__type": "ListItem", "__tag": 4054, "children": [ { "__type": "Paragraph", "__tag": 4045, "children": [ { "__type": "InlineCode", "__tag": 4051, "value": "A.min(axis=1)" }, { "__type": "Text", "__tag": 4046, "value": " returns a 1D " }, { "__type": "InlineCode", "__tag": 4051, "value": "coo_array" }, { "__type": "Text", "__tag": 4046, "value": " of the min along axis 1. Some reductions return dense arrays/matrices instead of sparse ones:" } ] }, { "__type": "Code", "__tag": 4050, "value": "============ =========\nMethod Result\n============ =========\nsum(axis) dense\nmean(axis) dense\nargmin(axis) dense\nargmax(axis) dense\nmin(axis) sparse\nmax(axis) sparse\nnanmin(axis) sparse\nnanmax(axis) sparse\n============ =========", "execution_status": null } ] } ] }, { "__type": "Paragraph", "__tag": 4045, "children": [ { "__type": "Text", "__tag": 4046, "value": "Generally, 2D sparray inputs lead to 1D results. 2D spmatrix inputs lead to 2D results." } ] } ] }, { "__type": "ListItem", "__tag": 4054, "children": [ { "__type": "Paragraph", "__tag": 4045, "children": [ { "__type": "Text", "__tag": 4046, "value": "Some reductions return a scalar. Those should behave as they did before and shouldn’t need to be considered during migration. E.g. " }, { "__type": "InlineCode", "__tag": 4051, "value": "A.min()" } ] } ] } ] } ], "title": [ { "__type": "Text", "__tag": 4046, "value": "Details: shape changes and reductions" } ], "level": 1, "target": "sparse-migration-shapes-reductions" }, { "__type": "Section", "__tag": 4015, "children": [ { "__type": "BulletList", "__tag": 4053, "ordered": false, "start": 1, "children": [ { "__type": "ListItem", "__tag": 4054, "children": [ { "__type": "Paragraph", "__tag": 4045, "children": [ { "__type": "Text", "__tag": 4046, "value": "The methods " }, { "__type": "InlineCode", "__tag": 4051, "value": "get_shape" }, { "__type": "Text", "__tag": 4046, "value": ", " }, { "__type": "InlineCode", "__tag": 4051, "value": "getrow" }, { "__type": "Text", "__tag": 4046, "value": ", " }, { "__type": "InlineCode", "__tag": 4051, "value": "getcol" }, { "__type": "Text", "__tag": 4046, "value": ", " }, { "__type": "InlineCode", "__tag": 4051, "value": "asfptype" }, { "__type": "Text", "__tag": 4046, "value": ", " }, { "__type": "InlineCode", "__tag": 4051, "value": "getnnz" }, { "__type": "Text", "__tag": 4046, "value": ", " }, { "__type": "InlineCode", "__tag": 4051, "value": "getH" }, { "__type": "Text", "__tag": 4046, "value": " and the attributes " }, { "__type": "InlineCode", "__tag": 4051, "value": ".A" }, { "__type": "Text", "__tag": 4046, "value": " and " }, { "__type": "InlineCode", "__tag": 4051, "value": ".H" }, { "__type": "Text", "__tag": 4046, "value": " are only present on spmatrices, not sparrays. It is recommended that you replace usage of them with alternatives before starting the shift to sparray." } ] }, { "__type": "Blockquote", "__tag": 4059, "children": [ { "__type": "Code", "__tag": 4050, "value": "=============== ====================\nFunction Alternative\n=============== ====================\nM.get_shape() A.shape\nM.getformat() A.format\nM.asfptype(…) A.astype(…)\nM.getmaxprint() A.maxprint\nM.getnnz() A.nnz\nM.getnnz(axis) A.count_nonzero(axis)\nM.getH() A.conj().T\nM.getrow(i) A[i, :]\nM.getcol(j) A[:, j]\nM.A A.toarray()\nM.H A.conj().T\n=============== ====================", "execution_status": null } ] } ] }, { "__type": "ListItem", "__tag": 4054, "children": [ { "__type": "Paragraph", "__tag": 4045, "children": [ { "__type": "Text", "__tag": 4046, "value": "Shape assignment (" }, { "__type": "InlineCode", "__tag": 4051, "value": "M.shape = (2, 6)" }, { "__type": "Text", "__tag": 4046, "value": ") is not permitted for sparray. Instead you should use " }, { "__type": "InlineCode", "__tag": 4051, "value": "A.reshape" }, { "__type": "Text", "__tag": 4046, "value": "." } ] } ] }, { "__type": "ListItem", "__tag": 4054, "children": [ { "__type": "Paragraph", "__tag": 4045, "children": [ { "__type": "InlineCode", "__tag": 4051, "value": "M.getnnz()" }, { "__type": "Text", "__tag": 4046, "value": " returns the number of stored values – not the number of non-zeros. " }, { "__type": "InlineCode", "__tag": 4051, "value": "A.nnz" }, { "__type": "Text", "__tag": 4046, "value": " does the same. To get the number of non-zeros, use " }, { "__type": "InlineCode", "__tag": 4051, "value": "A.count_nonzero()" }, { "__type": "Text", "__tag": 4046, "value": ". This is not new to the migration, but can be confusing." } ] }, { "__type": "Paragraph", "__tag": 4045, "children": [ { "__type": "Text", "__tag": 4046, "value": "To migrate from the " }, { "__type": "InlineCode", "__tag": 4051, "value": "axis" }, { "__type": "Text", "__tag": 4046, "value": " parameter of " }, { "__type": "InlineCode", "__tag": 4051, "value": "M.getnnz(axis=...)" }, { "__type": "Text", "__tag": 4046, "value": ", you can use " }, { "__type": "InlineCode", "__tag": 4051, "value": "A.count_nonzero(axis=...)" }, { "__type": "Text", "__tag": 4046, "value": " but it is not an exact replacement because it counts nonzero values instead of stored values. The difference is the number of explicitly stored zero values. If you really want the number of stored values by axis you will need to use some numpy tools." } ] }, { "__type": "Paragraph", "__tag": 4045, "children": [ { "__type": "Text", "__tag": 4046, "value": "The numpy tools approach works for COO, CSR, CSC formats, so convert to one of them. For CSR and CSC, the major axis is compressed and " }, { "__type": "InlineCode", "__tag": 4051, "value": "np.diff(A.indptr)" }, { "__type": "Text", "__tag": 4046, "value": " returns a dense 1D array with the number of stored values for each major axis value (row for CSR and column for CSC). The minor axes can be computed using " }, { "__type": "InlineCode", "__tag": 4051, "value": "np.bincount(A.indices, minlength=N)" }, { "__type": "Text", "__tag": 4046, "value": " where " }, { "__type": "InlineCode", "__tag": 4051, "value": "N" }, { "__type": "Text", "__tag": 4046, "value": " is the length of the minor axis (e.g. " }, { "__type": "InlineCode", "__tag": 4051, "value": "A.shape[1]" }, { "__type": "Text", "__tag": 4046, "value": " for CSR). The " }, { "__type": "InlineCode", "__tag": 4051, "value": "bincount" }, { "__type": "Text", "__tag": 4046, "value": " function works for any axis of COO format using " }, { "__type": "InlineCode", "__tag": 4051, "value": "A.coords[axis]" }, { "__type": "Text", "__tag": 4046, "value": " in place of " }, { "__type": "InlineCode", "__tag": 4051, "value": "A.indices" }, { "__type": "Text", "__tag": 4046, "value": "." } ] } ] } ] } ], "title": [ { "__type": "Text", "__tag": 4046, "value": "Removed methods and attributes" } ], "level": 1, "target": "sparse-migration-removed-methods" }, { "__type": "Section", "__tag": 4015, "children": [ { "__type": "BulletList", "__tag": 4053, "ordered": false, "start": 1, "children": [ { "__type": "ListItem", "__tag": 4054, "children": [ { "__type": "Paragraph", "__tag": 4045, "children": [ { "__type": "Text", "__tag": 4046, "value": "It can be tricky to distinguish scalar multiplication " }, { "__type": "InlineCode", "__tag": 4051, "value": "*" }, { "__type": "Text", "__tag": 4046, "value": " from matrix multiplciation " }, { "__type": "InlineCode", "__tag": 4051, "value": "*" }, { "__type": "Text", "__tag": 4046, "value": " as you migrate your code. Python solved this, in theory, by introducing the matrix multiplication operator " }, { "__type": "InlineCode", "__tag": 4051, "value": "@" }, { "__type": "Text", "__tag": 4046, "value": ". " }, { "__type": "InlineCode", "__tag": 4051, "value": "*" }, { "__type": "Text", "__tag": 4046, "value": " is used for scalar multiplication while " }, { "__type": "InlineCode", "__tag": 4051, "value": "@" }, { "__type": "Text", "__tag": 4046, "value": " for matrix multiplication. But converting expressions that use " }, { "__type": "InlineCode", "__tag": 4051, "value": "*" }, { "__type": "Text", "__tag": 4046, "value": " for both can be tricky and cause eye strain. Luckily, if your code has a test suite that covers the expressions you need to convert, you can use it to find places where " }, { "__type": "InlineCode", "__tag": 4051, "value": "*" }, { "__type": "Text", "__tag": 4046, "value": " is being used for matrix multiplication involving sparse matrices. Change those to " }, { "__type": "InlineCode", "__tag": 4051, "value": "@" }, { "__type": "Text", "__tag": 4046, "value": "." } ] }, { "__type": "Paragraph", "__tag": 4045, "children": [ { "__type": "Text", "__tag": 4046, "value": "The approach monkey-patches the spmatrix class dunder methods to raise an exception when " }, { "__type": "InlineCode", "__tag": 4051, "value": "*" }, { "__type": "Text", "__tag": 4046, "value": " is used for matrix multiplication (and not raise for scalar multiplication). The test suite will flag a failure at these locations. And a test failure is a success here because it shows where to make changes. Change the offending " }, { "__type": "InlineCode", "__tag": 4051, "value": "*" }, { "__type": "Text", "__tag": 4046, "value": " to " }, { "__type": "InlineCode", "__tag": 4051, "value": "@" }, { "__type": "Text", "__tag": 4046, "value": ", look nearby for other similar changes, and run the tests again. Similarly, this approach helps find where " }, { "__type": "InlineCode", "__tag": 4051, "value": "**" }, { "__type": "Text", "__tag": 4046, "value": " is used for matrix power. SciPy raises an exception when " }, { "__type": "InlineCode", "__tag": 4051, "value": "@" }, { "__type": "Text", "__tag": 4046, "value": " is used with for scalar multiplication, so that will catch places where you change when you shouldn't have. So the test suite with this monkey-patch checks the corrections too." } ] }, { "__type": "Paragraph", "__tag": 4045, "children": [ { "__type": "Text", "__tag": 4046, "value": "Add the following code to your " }, { "__type": "InlineCode", "__tag": 4051, "value": "conftest.py" }, { "__type": "Text", "__tag": 4046, "value": " file. Then run your tests locally. If there are many matrix expressions, you might want to test one section of your codebase at a time. A quick read of the code shows that it raises a " }, { "__type": "InlineCode", "__tag": 4051, "value": "ValueError" }, { "__type": "Text", "__tag": 4046, "value": " whenever " }, { "__type": "InlineCode", "__tag": 4051, "value": "*" }, { "__type": "Text", "__tag": 4046, "value": " is used between two matrix-like objects (sparse or dense), and whenever " }, { "__type": "InlineCode", "__tag": 4051, "value": "**" }, { "__type": "Text", "__tag": 4046, "value": " is used for matrix power. It also produces a warning whenever sum/mean/min/max/argmin/argmax are used with an axis so the output will be 2D with spmatrix and 1D with sparray. That means you check that the code will handle either 1D or 2D output via " }, { "__type": "InlineCode", "__tag": 4051, "value": "flatten" }, { "__type": "Text", "__tag": 4046, "value": "/" }, { "__type": "InlineCode", "__tag": 4051, "value": "ravel" }, { "__type": "Text", "__tag": 4046, "value": ", " }, { "__type": "InlineCode", "__tag": 4051, "value": "np.atleast_2d" }, { "__type": "Text", "__tag": 4046, "value": " or indexing." } ] }, { "__type": "Code", "__tag": 4050, "value": "#================== Added to check spmatrix usage ========================\n import scipy\n from warnings import warn\n\n def flag_this_call(*args, **kwds):\n raise ValueError(\"Old spmatrix function names for rand/spdiags called\")\n\n scipy.sparse._construct.rand = flag_this_call\n scipy.sparse._construct.spdiags = flag_this_call\n\n class _strict_mul_mixin:\n def __mul__(self, other):\n if not scipy.sparse._sputils.isscalarlike(other):\n raise ValueError('Operator * used here! Change to @?')\n return super().__mul__(other)\n\n def __rmul__(self, other):\n if not scipy.sparse._sputils.isscalarlike(other):\n raise ValueError('Operator * used here! Change to @?')\n return super().__rmul__(other)\n\n def __imul__(self, other):\n if not scipy.sparse._sputils.isscalarlike(other):\n raise ValueError('Operator * used here! Change to @?')\n return super().__imul__(other)\n\n def __pow__(self, *args, **kwargs):\n raise ValueError('spmatrix ** found! Use linalg.matrix_power?')\n\n @property\n def A(self):\n raise TypeError('spmatrix A property found! Use .toarray()')\n\n @property\n def H(self):\n raise TypeError('spmatrix H property found! Use .conjugate().T')\n\n def asfptype(self):\n raise TypeError('spmatrix asfptype found! rewrite needed')\n\n def get_shape(self):\n raise TypeError('spmatrix get_shape found! Use .shape')\n\n def getformat(self):\n raise TypeError('spmatrix getformat found! Use .format')\n\n def getmaxprint(self):\n raise TypeError('spmatrix getmaxprint found! Use .shape')\n\n def getnnz(self):\n raise TypeError('spmatrix getnnz found! Use .nnz')\n\n def getH(self):\n raise TypeError('spmatrix getH found! Use .conjugate().T')\n\n def getrow(self):\n raise TypeError('spmatrix getrow found! Use .row')\n\n def getcol(self):\n raise TypeError('spmatrix getcol found! Use .col')\n\n def sum(self, *args, **kwds):\n axis = args[0] if len(args)==1 else args if args else kwds.get(\"axis\", None)\n if axis is not None:\n warn(f\"\\nMIGRATION WARNING: spmatrix sum found using axis={axis}. \"\n \"\\nsparray with a single axis will produce 1D output. \"\n \"\\nCheck nearby to ensure 1D output is handled OK in this spot.\\n\")\n print(f\"{args=} {axis=} {kwds=}\")\n return super().sum(*args, **kwds)\n\n def mean(self, *args, **kwds):\n axis = args[0] if len(args)==1 else args if args else kwds.get(\"axis\", None)\n if axis is not None:\n warn(f\"\\nMIGRATION WARNING: spmatrix mean found using axis={axis}.\"\n \"\\nsparray with a single axis will produce 1D output.\\n\"\n \"Check nearby to ensure 1D output is handled OK in this spot.\\n\")\n return super().mean(*args, **kwds)\n\n def min(self, *args, **kwds):\n axis = args[0] if len(args)==1 else args if args else kwds.get(\"axis\", None)\n if axis is not None:\n warn(f\"\\nMIGRATION WARNING: spmatrix min found using axis={axis}.\"\n \"\\nsparray with a single axis will produce 1D output. \"\n \"Check nearby to ensure 1D output is handled OK in this spot.\\n\")\n return super().min(*args, **kwds)\n\n def max(self, *args, **kwds):\n axis = args[0] if len(args)==1 else args if args else kwds.get(\"axis\", None)\n if axis is not None:\n warn(f\"\\nMIGRATION WARNING: spmatrix max found using axis={axis}.\"\n \"\\nsparray with a single axis will produce 1D output. \"\n \"Check nearby to ensure 1D output is handled OK in this spot.\\n\")\n return super().max(*args, **kwds)\n\n def argmin(self, *args, **kwds):\n axis = args[0] if len(args)==1 else args if args else kwds.get(\"axis\", None)\n if axis is not None:\n warn(f\"\\nMIGRATION WARNING: spmatrix argmin found using axis={axis}.\"\n \"\\nsparray with a single axis will produce 1D output. \"\n \"Check nearby to ensure 1D output is handled OK in this spot.\\n\")\n return super().argmin(*args, **kwds)\n\n def argmax(self, *args, **kwds):\n axis = args[0] if len(args)==1 else args if args else kwds.get(\"axis\", None)\n if axis is not None:\n warn(f\"\\nMIGRATION WARNING: spmatrix argmax found using axis={axis}.\"\n \"\\nsparray with a single axis will produce 1D output. \"\n \"Check nearby to ensure 1D output is handled OK in this spot.\\n\")\n return super().argmax(*args, **kwds)\n\n\n class coo_matrix_strict(_strict_mul_mixin, scipy.sparse.coo_matrix):\n pass\n\n class bsr_matrix_strict(_strict_mul_mixin, scipy.sparse.bsr_matrix):\n pass\n\n class csr_matrix_strict(_strict_mul_mixin, scipy.sparse.csr_matrix):\n pass\n\n class csc_matrix_strict(_strict_mul_mixin, scipy.sparse.csc_matrix):\n pass\n\n class dok_matrix_strict(_strict_mul_mixin, scipy.sparse.dok_matrix):\n pass\n\n class lil_matrix_strict(_strict_mul_mixin, scipy.sparse.lil_matrix):\n pass\n\n class dia_matrix_strict(_strict_mul_mixin, scipy.sparse.dia_matrix):\n pass\n\n scipy.sparse.coo_matrix = scipy.sparse._coo.coo_matrix = coo_matrix_strict\n scipy.sparse.bsr_matrix = scipy.sparse._bsr.bsr_matrix = bsr_matrix_strict\n scipy.sparse.csr_matrix = scipy.sparse._csr.csr_matrix = csr_matrix_strict\n scipy.sparse.csc_matrix = scipy.sparse._csc.csc_matrix = csc_matrix_strict\n scipy.sparse.dok_matrix = scipy.sparse._dok.dok_matrix = dok_matrix_strict\n scipy.sparse.lil_matrix = scipy.sparse._lil.lil_matrix = lil_matrix_strict\n scipy.sparse.dia_matrix = scipy.sparse._dia.dia_matrix = dia_matrix_strict\n\n scipy.sparse._compressed.csr_matrix = csr_matrix_strict\n\n scipy.sparse._construct.bsr_matrix = bsr_matrix_strict\n scipy.sparse._construct.coo_matrix = coo_matrix_strict\n scipy.sparse._construct.csc_matrix = csc_matrix_strict\n scipy.sparse._construct.csr_matrix = csr_matrix_strict\n scipy.sparse._construct.dia_matrix = dia_matrix_strict\n\n scipy.sparse._extract.coo_matrix = coo_matrix_strict\n\n scipy.sparse._matrix.bsr_matrix = bsr_matrix_strict\n scipy.sparse._matrix.coo_matrix = coo_matrix_strict\n scipy.sparse._matrix.csc_matrix = csc_matrix_strict\n scipy.sparse._matrix.csr_matrix = csr_matrix_strict\n scipy.sparse._matrix.dia_matrix = dia_matrix_strict\n scipy.sparse._matrix.dok_matrix = dok_matrix_strict\n scipy.sparse._matrix.lil_matrix = lil_matrix_strict\n\n del coo_matrix_strict\n del bsr_matrix_strict\n del csr_matrix_strict\n del csc_matrix_strict\n del dok_matrix_strict\n del lil_matrix_strict\n del dia_matrix_strict\n #==========================================", "execution_status": null } ] } ] } ], "title": [ { "__type": "Text", "__tag": 4046, "value": "Use tests to find * and ** spots" } ], "level": 1, "target": "sparse-migration-star-vs-at" }, { "__type": "Section", "__tag": 4015, "children": [ { "__type": "Paragraph", "__tag": 4045, "children": [ { "__type": "Text", "__tag": 4046, "value": "If you provide compressed indices to a constructor, e.g. " }, { "__type": "InlineCode", "__tag": 4051, "value": "csr_array((data, indices, indptr))" }, { "__type": "Text", "__tag": 4046, "value": " sparse arrays set the index dtype by only checking the index arrays dtype, while sparse matrices check the index values too and may downcast to int32 (see " }, { "__type": "Link", "__tag": 4049, "children": [ { "__type": "Text", "__tag": 4046, "value": "gh-18509" } ], "url": "https://github.com/scipy/scipy/pull/18509", "title": "" }, { "__type": "Text", "__tag": 4046, "value": " for more details). This means you may get int64 indexing when you used to get int32. You can control this by setting the " }, { "__type": "InlineCode", "__tag": 4051, "value": "dtype" }, { "__type": "Text", "__tag": 4046, "value": " before instantiating, or by recasting after construction." } ] }, { "__type": "Paragraph", "__tag": 4045, "children": [ { "__type": "Text", "__tag": 4046, "value": "Two sparse utility functions can help with handling the index dtype. Use " }, { "__type": "InlineCode", "__tag": 4051, "value": "get_index_dtype(arrays, maxval, check_contents)" }, { "__type": "Text", "__tag": 4046, "value": " while creating indices to find an appropriate dtype (int32 or int64) to use for your compressed indices." } ] }, { "__type": "Paragraph", "__tag": 4045, "children": [ { "__type": "Text", "__tag": 4046, "value": "Use " }, { "__type": "InlineCode", "__tag": 4051, "value": "safely_cast_index_arrays(A, idx_dtype)" }, { "__type": "Text", "__tag": 4046, "value": " for recasting after construction, while making sure you con't create overflows during downcasting. This function doesn't actually change the input array. The cast arrays are returned. And copies are only made when needed. So you can check if casting was done using " }, { "__type": "InlineCode", "__tag": 4051, "value": "if indices is not A.indices:" } ] }, { "__type": "Paragraph", "__tag": 4045, "children": [ { "__type": "Text", "__tag": 4046, "value": "The function signatures are " } ] }, { "__type": "Code", "__tag": 4050, "value": "def get_index_dtype(arrays=(), maxval=None, check_contents=False):\ndef safely_cast_index_arrays(A, idx_dtype=np.int32, msg=\"\"):", "execution_status": null }, { "__type": "Paragraph", "__tag": 4045, "children": [ { "__type": "Text", "__tag": 4046, "value": "Example idioms include the following for " }, { "__type": "InlineCode", "__tag": 4051, "value": "get_index_dtype" }, { "__type": "Text", "__tag": 4046, "value": " " } ] }, { "__type": "Code", "__tag": 4050, "value": ".. code-block:: python\n\n # select index dtype before construction based on shape\n shape = (3, 3)\n idx_dtype = scipy.sparse.get_index_dtype(maxval=max(shape))\n indices = np.array([0, 1, 0], dtype=idx_dtype)\n indptr = np.arange(3, dtype=idx_dtype)\n A = csr_array((data, indices, indptr), shape=shape)", "execution_status": null }, { "__type": "Paragraph", "__tag": 4045, "children": [ { "__type": "Text", "__tag": 4046, "value": "and for " }, { "__type": "InlineCode", "__tag": 4051, "value": "safely_cast_index_arrays" }, { "__type": "Text", "__tag": 4046, "value": " " } ] }, { "__type": "Code", "__tag": 4050, "value": ".. code-block:: python\n\n # rescast after construction, raising exception if shape too big\n indices, indptr = scipy.sparse.safely_cast_index_arrays(B, np.int32)\n B.indices, B.indptr = indices, indptr", "execution_status": null } ], "title": [ { "__type": "Text", "__tag": 4046, "value": "Index Array DTypes" } ], "level": 1, "target": "sparse-migration-index-array-dtypes" }, { "__type": "Section", "__tag": 4015, "children": [ { "__type": "BulletList", "__tag": 4053, "ordered": false, "start": 1, "children": [ { "__type": "ListItem", "__tag": 4054, "children": [ { "__type": "Paragraph", "__tag": 4045, "children": [ { "__type": "Text", "__tag": 4046, "value": "Binary operators " }, { "__type": "InlineCode", "__tag": 4051, "value": "+, -, *, /, @, !=, >" }, { "__type": "Text", "__tag": 4046, "value": " act on sparse and/or dense operands:" } ] }, { "__type": "BulletList", "__tag": 4053, "ordered": false, "start": 1, "children": [ { "__type": "ListItem", "__tag": 4054, "children": [ { "__type": "Paragraph", "__tag": 4045, "children": [ { "__type": "Text", "__tag": 4046, "value": "If all inputs are sparse, the output is usually sparse as well. The exception being " }, { "__type": "InlineCode", "__tag": 4051, "value": "/" }, { "__type": "Text", "__tag": 4046, "value": " which returns dense (dividing by the default value " }, { "__type": "InlineCode", "__tag": 4051, "value": "0" }, { "__type": "Text", "__tag": 4046, "value": " is " }, { "__type": "InlineCode", "__tag": 4051, "value": "nan" }, { "__type": "Text", "__tag": 4046, "value": ")." } ] } ] }, { "__type": "ListItem", "__tag": 4054, "children": [ { "__type": "Paragraph", "__tag": 4045, "children": [ { "__type": "Text", "__tag": 4046, "value": "If inputs are mixed sparse and dense, the result is usually dense (i.e., " }, { "__type": "InlineCode", "__tag": 4051, "value": "np.ndarray" }, { "__type": "Text", "__tag": 4046, "value": "). Exceptions are " }, { "__type": "InlineCode", "__tag": 4051, "value": "*" }, { "__type": "Text", "__tag": 4046, "value": " which is sparse, and " }, { "__type": "InlineCode", "__tag": 4051, "value": "/" }, { "__type": "Text", "__tag": 4046, "value": " which is not implemented for " }, { "__type": "InlineCode", "__tag": 4051, "value": "dense / sparse" }, { "__type": "Text", "__tag": 4046, "value": ", and returns sparse for " }, { "__type": "InlineCode", "__tag": 4051, "value": "sparse / dense" }, { "__type": "Text", "__tag": 4046, "value": "." } ] } ] } ] } ] }, { "__type": "ListItem", "__tag": 4054, "children": [ { "__type": "Paragraph", "__tag": 4045, "children": [ { "__type": "Text", "__tag": 4046, "value": "Binary operators " }, { "__type": "InlineCode", "__tag": 4051, "value": "+, -, *, /, @, !=, >" }, { "__type": "Text", "__tag": 4046, "value": " with array and/or matrix operands:" } ] }, { "__type": "BulletList", "__tag": 4053, "ordered": false, "start": 1, "children": [ { "__type": "ListItem", "__tag": 4054, "children": [ { "__type": "Paragraph", "__tag": 4045, "children": [ { "__type": "Text", "__tag": 4046, "value": "If all inputs are arrays, the outputs are arrays and the same is true for matrices." } ] } ] }, { "__type": "ListItem", "__tag": 4054, "children": [ { "__type": "Paragraph", "__tag": 4045, "children": [ { "__type": "Text", "__tag": 4046, "value": "When mixing sparse arrays with sparse matrices, the leading operand provides the type for the output, e.g. " }, { "__type": "InlineCode", "__tag": 4051, "value": "sparray + spmatrix" }, { "__type": "Text", "__tag": 4046, "value": " gives a sparse array while reversing the order gives a sparse matrix." } ] } ] }, { "__type": "ListItem", "__tag": 4054, "children": [ { "__type": "Paragraph", "__tag": 4045, "children": [ { "__type": "Text", "__tag": 4046, "value": "When mixing dense matrices with sparse arrays, the results are usually arrays with exceptions for comparisons, e.g. " }, { "__type": "InlineCode", "__tag": 4051, "value": ">" }, { "__type": "Text", "__tag": 4046, "value": " which return dense matrices." } ] } ] }, { "__type": "ListItem", "__tag": 4054, "children": [ { "__type": "Paragraph", "__tag": 4045, "children": [ { "__type": "Text", "__tag": 4046, "value": "When mixing dense arrays with sparse matrices, the results are usually matrices with an exception for " }, { "__type": "InlineCode", "__tag": 4051, "value": "array @ sparse matrix" }, { "__type": "Text", "__tag": 4046, "value": " which returns a dense array." } ] } ] } ] } ] } ] } ], "title": [ { "__type": "Text", "__tag": 4046, "value": "Other" } ], "level": 1, "target": null } ], "local_refs": [] }