# repos

node_id: R_kgDOGGsxew
name: datasette-template-request
full_name: simonw/datasette-template-request
html_url: https://github.com/simonw/datasette-template-request
description: Expose the Datasette request object to custom templates
created_at: 2021-09-23T17:07:00Z
updated_at: 2024-10-09T03:47:37Z
pushed_at: 2021-09-23T17:29:36Z
homepage: https://datasette.io/plugins/datasette-template-request
language: Python
topics: ["datasette", "datasette-io", "datasette-plugin"]
visibility: public
default_branch: main
permissions: {"admin": false, "maintain": false, "push": false, "triage": false, "pull": true}
template_repository:
  {"id": 400878073, "node_id": "MDEwOlJlcG9zaXRvcnk0MDA4NzgwNzM=", "name": "datasette-plugin-template-repository", "full_name": "simonw/datasette-plugin-template-repository", "private": false, "owner": {"login": "simonw", "id": 9599, "node_id": "MDQ6VXNlcjk1OTk=", "avatar_url": "https://avatars.githubusercontent.com/u/9599?v=4", "gravatar_id": "", "url": "https://api.github.com/users/simonw", "html_url": "https://github.com/simonw", "followers_url": "https://api.github.com/users/simonw/followers", "following_url": "https://api.github.com/users/simonw/following{/other_user}", "gists_url": "https://api.github.com/users/simonw/gists{/gist_id}", "starred_url": "https://api.github.com/users/simonw/starred{/owner}{/repo}", "subscriptions_url": "https://api.github.com/users/simonw/subscriptions", "organizations_url": "https://api.github.com/users/simonw/orgs", "repos_url": "https://api.github.com/users/simonw/repos", "events_url": "https://api.github.com/users/simonw/events{/privacy}", "received_events_url": "https://api.github.com/users/simonw/received_events", "type": "User", "user_view_type": "public", "site_admin": false}, "html_url": "https://github.com/simonw/datasette-plugin-template-repository", "description": "GitHub template repository for creating new Datasette plugins, using the simonw/datasette-plugin cookiecutter template", "fork": false, "url": "https://api.github.com/repos/simonw/datasette-plugin-template-repository", "forks_url": "https://api.github.com/repos/simonw/datasette-plugin-template-repository/forks", "keys_url": "https://api.github.com/repos/simonw/datasette-plugin-template-repository/keys{/key_id}", "collaborators_url": "https://api.github.com/repos/simonw/datasette-plugin-template-repository/collaborators{/collaborator}", "teams_url": "https://api.github.com/repos/simonw/datasette-plugin-template-repository/teams", "hooks_url": "https://api.github.com/repos/simonw/datasette-plugin-template-repository/hooks", "issue_events_url": "https://api.github.com/repos/simonw/datasette-plugin-template-repository/issues/events{/number}", "events_url": "https://api.github.com/repos/simonw/datasette-plugin-template-repository/events", "assignees_url": "https://api.github.com/repos/simonw/datasette-plugin-template-repository/assignees{/user}", "branches_url": "https://api.github.com/repos/simonw/datasette-plugin-template-repository/branches{/branch}", "tags_url": "https://api.github.com/repos/simonw/datasette-plugin-template-repository/tags", "blobs_url": "https://api.github.com/repos/simonw/datasette-plugin-template-repository/git/blobs{/sha}", "git_tags_url": "https://api.github.com/repos/simonw/datasette-plugin-template-repository/git/tags{/sha}", "git_refs_url": "https://api.github.com/repos/simonw/datasette-plugin-template-repository/git/refs{/sha}", "trees_url": "https://api.github.com/repos/simonw/datasette-plugin-template-repository/git/trees{/sha}", "statuses_url": "https://api.github.com/repos/simonw/datasette-plugin-template-repository/statuses/{sha}", "languages_url": "https://api.github.com/repos/simonw/datasette-plugin-template-repository/languages", "stargazers_url": "https://api.github.com/repos/simonw/datasette-plugin-template-repository/stargazers", "contributors_url": "https://api.github.com/repos/simonw/datasette-plugin-template-repository/contributors", "subscribers_url": "https://api.github.com/repos/simonw/datasette-plugin-template-repository/subscribers", "subscription_url": "https://api.github.com/repos/simonw/datasette-plugin-template-repository/subscription", "commits_url": "https://api.github.com/repos/simonw/datasette-plugin-template-repository/commits{/sha}", "git_commits_url": "https://api.github.com/repos/simonw/datasette-plugin-template-repository/git/commits{/sha}", "comments_url": "https://api.github.com/repos/simonw/datasette-plugin-template-repository/comments{/number}", "issue_comment_url": "https://api.github.com/repos/simonw/datasette-plugin-template-repository/issues/comments{/number}", "contents_url": "https://api.github.com/repos/simonw/datasette-plugin-template-repository/contents/{+path}", "compare_url": "https://api.github.com/repos/simonw/datasette-plugin-template-repository/compare/{base}...{head}", "merges_url": "https://api.github.com/repos/simonw/datasette-plugin-template-repository/merges", "archive_url": "https://api.github.com/repos/simonw/datasette-plugin-template-repository/{archive_format}{/ref}", "downloads_url": "https://api.github.com/repos/simonw/datasette-plugin-template-repository/downloads", "issues_url": "https://api.github.com/repos/simonw/datasette-plugin-template-repository/issues{/number}", "pulls_url": "https://api.github.com/repos/simonw/datasette-plugin-template-repository/pulls{/number}", "milestones_url": "https://api.github.com/repos/simonw/datasette-plugin-template-repository/milestones{/number}", "notifications_url": "https://api.github.com/repos/simonw/datasette-plugin-template-repository/notifications{?since,all,participating}", "labels_url": "https://api.github.com/repos/simonw/datasette-plugin-template-repository/labels{/name}", "releases_url": "https://api.github.com/repos/simonw/datasette-plugin-template-repository/releases{/id}", "deployments_url": "https://api.github.com/repos/simonw/datasette-plugin-template-repository/deployments", "created_at": "2021-08-28T19:50:28Z", "updated_at": "2025-02-18T20:42:29Z", "pushed_at": "2025-02-18T20:42:25Z", "git_url": "git://github.com/simonw/datasette-plugin-template-repository.git", "ssh_url": "git@github.com:simonw/datasette-plugin-template-repository.git", "clone_url": "https://github.com/simonw/datasette-plugin-template-repository.git", "svn_url": "https://github.com/simonw/datasette-plugin-template-repository", "homepage": "", "size": 24, "stargazers_count": 24, "watchers_count": 24, "language": null, "has_issues": true, "has_projects": true, "has_downloads": true, "has_wiki": true, "has_pages": false, "has_discussions": false, "forks_count": 0, "mirror_url": null, "archived": false, "disabled": false, "open_issues_count": 0, "license": null, "allow_forking": true, "is_template": true, "web_commit_signoff_required": false, "topics": [], "visibility": "public", "forks": 0, "open_issues": 0, "watchers": 24, "default_branch": "main", "permissions": {"admin": false, "maintain": false, "push": false, "triage": false, "pull": true}}

readme:
  # datasette-template-request

  [![PyPI](https://img.shields.io/pypi/v/datasette-template-request.svg)](https://pypi.org/project/datasette-template-request/)
  [![Changelog](https://img.shields.io/github/v/release/simonw/datasette-template-request?include_prereleases&label=changelog)](https://github.com/simonw/datasette-template-request/releases)
  [![Tests](https://github.com/simonw/datasette-template-request/workflows/Test/badge.svg)](https://github.com/simonw/datasette-template-request/actions?query=workflow%3ATest)
  [![License](https://img.shields.io/badge/license-Apache%202.0-blue.svg)](https://github.com/simonw/datasette-template-request/blob/main/LICENSE)

  Expose the Datasette request object to custom templates

  ## Installation

  Install this plugin in the same environment as Datasette.

      $ datasette install datasette-template-request

  ## Usage

  Once this plugin is installed, Datasette [custom templates](https://docs.datasette.io/en/stable/custom_templates.html) can use `{{ request }}` to access the current [request object](https://docs.datasette.io/en/stable/internals.html#request-object). For example, to access `?name=Cleo` in the query string a template could use this:

      Name: {{ request.args.name }}

  ## Development

  To set up this plugin locally, first checkout the code. Then create a new virtual environment:

      cd datasette-template-request
      python3 -mvenv venv
      source venv/bin/activate

  Or if you are using `pipenv`:

      pipenv shell

  Now install the dependencies and test dependencies:

      pip install -e '.[test]'

  To run the tests:

      pytest

readme_html:
  <div id="readme" class="md" data-path="README.md"><article class="markdown-body entry-content container-lg" itemprop="text"><div class="markdown-heading" dir="auto"><h1 class="heading-element" dir="auto">datasette-template-request</h1><a id="user-content-datasette-template-request" class="anchor" aria-label="Permalink: datasette-template-request" href="#datasette-template-request"><svg class="octicon octicon-link" viewbox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
  <p dir="auto"><a href="https://pypi.org/project/datasette-template-request/" rel="nofollow"><img src="https://img.shields.io/pypi/v/datasette-template-request.svg" alt="PyPI" style="max-width: 100%;" data-camo-src="https://camo.githubusercontent.com/4af6b0374e59ae533a4aa83b50849eb30aca3e42a79f7554d2c00a1db0140444/68747470733a2f2f696d672e736869656c64732e696f2f707970692f762f6461746173657474652d74656d706c6174652d726571756573742e737667"></a>
  <a href="https://github.com/simonw/datasette-template-request/releases"><img src="https://img.shields.io/github/v/release/simonw/datasette-template-request?include_prereleases&label=changelog" alt="Changelog" style="max-width: 100%;" data-camo-src="https://camo.githubusercontent.com/7133ea2765e27974a8e1277e5ce7692b1d6881e732cf699c63265aab4e2ab817/68747470733a2f2f696d672e736869656c64732e696f2f6769746875622f762f72656c656173652f73696d6f6e772f6461746173657474652d74656d706c6174652d726571756573743f696e636c7564655f70726572656c6561736573266c6162656c3d6368616e67656c6f67"></a>
  <a href="https://github.com/simonw/datasette-template-request/actions?query=workflow%3ATest"><img src="https://github.com/simonw/datasette-template-request/workflows/Test/badge.svg" alt="Tests" style="max-width: 100%;"></a>
  <a href="https://github.com/simonw/datasette-template-request/blob/main/LICENSE"><img src="https://img.shields.io/badge/license-Apache%202.0-blue.svg" alt="License" style="max-width: 100%;" data-camo-src="https://camo.githubusercontent.com/c355f200ea90fddaa407b6eaab303663a669248ea3ca7b1fcf77dbe04ff5f48c/68747470733a2f2f696d672e736869656c64732e696f2f62616467652f6c6963656e73652d417061636865253230322e302d626c75652e737667"></a></p>
  <p dir="auto">Expose the Datasette request object to custom templates</p>
  <div class="markdown-heading" dir="auto"><h2 class="heading-element" dir="auto">Installation</h2><a id="user-content-installation" class="anchor" aria-label="Permalink: Installation" href="#installation"><svg class="octicon octicon-link" viewbox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
  <p dir="auto">Install this plugin in the same environment as Datasette.</p>
  <div class="snippet-clipboard-content notranslate position-relative overflow-auto" data-snippet-clipboard-copy-content="$ datasette install datasette-template-request"><pre class="notranslate"><code>$ datasette install datasette-template-request
  </code></pre></div>
  <div class="markdown-heading" dir="auto"><h2 class="heading-element" dir="auto">Usage</h2><a id="user-content-usage" class="anchor" aria-label="Permalink: Usage" href="#usage"><svg class="octicon octicon-link" viewbox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
  <p dir="auto">Once this plugin is installed, Datasette <a href="https://docs.datasette.io/en/stable/custom_templates.html" rel="nofollow">custom templates</a> can use <code>{{ request }}</code> to access the current <a href="https://docs.datasette.io/en/stable/internals.html#request-object" rel="nofollow">request object</a>. For example, to access <code>?name=Cleo</code> in the query string a template could use this:</p>
  <div class="snippet-clipboard-content notranslate position-relative overflow-auto" data-snippet-clipboard-copy-content="Name: {{ request.args.name }}"><pre class="notranslate"><code>Name: {{ request.args.name }}
  </code></pre></div>
  <div class="markdown-heading" dir="auto"><h2 class="heading-element" dir="auto">Development</h2><a id="user-content-development" class="anchor" aria-label="Permalink: Development" href="#development"><svg class="octicon octicon-link" viewbox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
  <p dir="auto">To set up this plugin locally, first checkout the code. Then create a new virtual environment:</p>
  <div class="snippet-clipboard-content notranslate position-relative overflow-auto" data-snippet-clipboard-copy-content="cd datasette-template-request
  python3 -mvenv venv
  source venv/bin/activate"><pre class="notranslate"><code>cd datasette-template-request
  python3 -mvenv venv
  source venv/bin/activate
  </code></pre></div>
  <p dir="auto">Or if you are using <code>pipenv</code>:</p>
  <div class="snippet-clipboard-content notranslate position-relative overflow-auto" data-snippet-clipboard-copy-content="pipenv shell"><pre class="notranslate"><code>pipenv shell
  </code></pre></div>
  <p dir="auto">Now install the dependencies and test dependencies:</p>
  <div class="snippet-clipboard-content notranslate position-relative overflow-auto" data-snippet-clipboard-copy-content="pip install -e '.[test]'"><pre class="notranslate"><code>pip install -e '.[test]'
  </code></pre></div>
  <p dir="auto">To run the tests:</p>
  <div class="snippet-clipboard-content notranslate position-relative overflow-auto" data-snippet-clipboard-copy-content="pytest"><pre class="notranslate"><code>pytest
  </code></pre></div>
  </article></div>


node_id: R_kgDOGM0DcA
name: sqlite-html
full_name: asg017/sqlite-html
html_url: https://github.com/asg017/sqlite-html
description: A SQLite extension for querying, manipulating, and creating HTML elements.
created_at: 2021-10-11T21:15:19Z
updated_at: 2025-04-25T18:16:16Z
pushed_at: 2023-08-06T01:24:57Z
language: Go
license: mit
topics: ["sqlite", "sqlite-extension"]
visibility: public
default_branch: main
permissions: {"admin": false, "maintain": false, "push": false, "triage": false, "pull": true}
readme:
  # sqlite-html

  A SQLite extension for querying, manipulating, and creating HTML elements.

  - Extract HTML or text from HTML with CSS selectors, like `.querySelector()`, `.innerHTML`, and `.innerText`
  - Generate a table of matching elements from a CSS selector, like `.querySelectorAll()`
  - Safely create HTML elements in a query, like `.createElement()` and `.appendChild()`

  `sqlite-html`'s API is modeled after the official [JSON1](https://www.sqlite.org/json1.html#jmini) SQLite extension.

  This extension is written in Go, thanks to [riyaz-ali/sqlite](https://github.com/riyaz-ali/sqlite). While this library aims to be fast and efficient, it is overall slower than what a pure C SQLite extension could be, but in practice you may not notice much of a difference.

  ## Usage

  ```sql
  .load ./html0
  select html_extract('<p> Anakin <b>Skywalker</b> </p>', 'b');
  -- "<b>Skywalker</b>"
  ```

  `sqlite-html` is similar to other HTML scraping tools like [BeautifulSoup](https://beautiful-soup-4.readthedocs.io/en/latest/) (Python) or [cheerio](https://cheerio.js.org/) (Node.js) or [nokogiri](https://nokogiri.org/) (Ruby). You can use CSS selectors to extract individual elements or groups of elements to query data from HTML sources.

  For example, here we find all `href` links in an `index.html` file.

  ```sql
  select
    text as name,
    html_attribute_get(anchors, 'a', 'href') as href
  from html_each(readfile('index.html'), 'a') as anchors
  ```

  We can also safely generate HTML with `html_element`, modeled after React's [`React.createElement`](https://reactjs.org/docs/react-api.html#createelement).

  ```sql
  select html_element('p', null,
    'Luke, I am your',
    html_element('b', null, 'father'),
    '!',

    html_element('img', json_object(
      'src', 'https://images.dog.ceo/breeds/groenendael/n02105056_4600.jpg',
      'width', 200
    ))
  );

  -- "<p>Luke, I am your<b>father</b>!<img src="https://images.dog.ceo/breeds/groenendael/n02105056_4600.jpg" width="200.000000"/></p>"
  ```

  ## Documentation

  See [`docs.md`](./docs.md) for a full API reference.

  ## Installing

  | Language       | Install                                                      |                                                                                                                                                                                             |
  | -------------- | ------------------------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
  | Python         | `pip install sqlite-html`                                    | [![PyPI](https://img.shields.io/pypi/v/sqlite-html.svg?color=blue&logo=python&logoColor=white)](https://pypi.org/project/sqlite-html/)                                                      |
  | Datasette      | `datasette install datasette-sqlite-html`                    | [![Datasette](https://img.shields.io/pypi/v/datasette-sqlite-html.svg?color=B6B6D9&label=Datasette+plugin&logoColor=white&logo=python)](https://datasette.io/plugins/datasette-sqlite-html) |
  | Node.js        | `npm install sqlite-html`                                    | [![npm](https://img.shields.io/npm/v/sqlite-html.svg?color=green&logo=nodedotjs&logoColor=white)](https://www.npmjs.com/package/sqlite-html)                                                |
  | Deno           | [`deno.land/x/sqlite_html`](https://deno.land/x/sqlite_html) | [![deno.land/x release](https://img.shields.io/github/v/release/asg017/sqlite-html?color=fef8d2&include_prereleases&label=deno.land%2Fx&logo=deno)](https://deno.land/x/sqlite_html)        |
  | Ruby           | `gem install sqlite-html`                                    | ![Gem](https://img.shields.io/gem/v/sqlite-html?color=red&logo=rubygems&logoColor=white)                                                                                                    |
  | Github Release |                                                              | ![GitHub tag (latest SemVer pre-release)](https://img.shields.io/github/v/tag/asg017/sqlite-html?color=lightgrey&include_prereleases&label=Github+release&logo=github)                      |

  <!--
  | Elixir         | [`hex.pm/packages/sqlite_html`](https://hex.pm/packages/sqlite_html) | [![Hex.pm](https://img.shields.io/hexpm/v/sqlite_html?color=purple&logo=elixir)](https://hex.pm/packages/sqlite_html)                                                                       |
  | Go             | `go get -u github.com/asg017/sqlite-html/bindings/go`               | [![Go Reference](https://pkg.go.dev/badge/github.com/asg017/sqlite-html/bindings/go.svg)](https://pkg.go.dev/github.com/asg017/sqlite-html/bindings/go)                                     |
  | Rust           | `cargo add sqlite-html`                                             | [![Crates.io](https://img.shields.io/crates/v/sqlite-html?logo=rust)](https://crates.io/crates/sqlite-html)                                                                                 |
  -->

  The [Releases page](https://github.com/asg017/sqlite-html/releases) contains pre-built binaries for Linux amd64, MacOS amd64 (no arm), and Windows.

  ### As a loadable extension

  If you want to use `sqlite-html` as a [Runtime-loadable extension](https://www.sqlite.org/loadext.html), Download the `html0.dylib` (for MacOS), `html0.so` (Linux), or `html0.dll` (Windows) file from a release and load it into your SQLite environment.

  > **Note:**
  > The `0` in the filename (`html0.dylib`/ `html0.so`/`html0.dll`) denotes the major version of `sqlite-html`. Currently `sqlite-html` is pre v1, so expect breaking changes in future versions.

  For example, if you are using the [SQLite CLI](https://www.sqlite.org/cli.html), you can load the library like so:

  ```sql
  .load ./html0
  select html_version();
  -- v0.0.1
  ```

  Or in Python, using the builtin [sqlite3 module](https://docs.python.org/3/library/sqlite3.html):

  ```python
  import sqlite3

  con = sqlite3.connect(":memory:")

  con.enable_load_extension(True)
  con.load_extension("./html0")

  print(con.execute("select html_version()").fetchone())
  # ('v0.0.1',)
  ```

  Or in Node.js using [better-sqlite3](https://github.com/WiseLibs/better-sqlite3):

  ```javascript
  const Database = require("better-sqlite3");
  const db = new Database(":memory:");

  db.loadExtension("./html0");

  console.log(db.prepare("select html_version()").get());
  // { 'html_version()': 'v0.0.1' }
  ```

  Or with [Datasette](https://datasette.io/):

  ```
  datasette data.db --load-extension ./html0
  ```

  ## See also

  - [sqlite-http](https://github.com/asg017/sqlite-http), for making HTTP requests in SQLite (pairs great with this tool)
  - [htmlq](https://github.com/mgdm/htmlq), for a similar but CLI-based HTML query tool using CSS selectors
  - [riyaz-ali/sqlite](https://github.com/riyaz-ali/sqlite), the brilliant Go library that this library depends on
  - [nalgeon/sqlean](https://github.com/nalgeon/sqlean), several pre-compiled handy SQLite functions, in C

readme_html:
  <div id="readme" class="md" data-path="README.md"><article class="markdown-body entry-content container-lg" itemprop="text"><div class="markdown-heading" dir="auto"><h1 class="heading-element" dir="auto">sqlite-html</h1><a id="user-content-sqlite-html" class="anchor" aria-label="Permalink: sqlite-html" href="#sqlite-html"><svg class="octicon octicon-link" viewbox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
  <p dir="auto">A SQLite extension for querying, manipulating, and creating HTML elements.</p>
  <ul dir="auto">
  <li>Extract HTML or text from HTML with CSS selectors, like <code>.querySelector()</code>, <code>.innerHTML</code>, and <code>.innerText</code></li>
  <li>Generate a table of matching elements from a CSS selector, like <code>.querySelectorAll()</code></li>
  <li>Safely create HTML elements in a query, like <code>.createElement()</code> and <code>.appendChild()</code></li>
  </ul>
  <p dir="auto"><code>sqlite-html</code>'s API is modeled after the official <a href="https://www.sqlite.org/json1.html#jmini" rel="nofollow">JSON1</a> SQLite extension.</p>
  <p dir="auto">This extension is written in Go, thanks to <a href="https://github.com/riyaz-ali/sqlite">riyaz-ali/sqlite</a>. While this library aims to be fast and efficient, it is overall slower than what a pure C SQLite extension could be, but in practice you may not notice much of a difference.</p>
  <div class="markdown-heading" dir="auto"><h2 class="heading-element" dir="auto">Usage</h2><a id="user-content-usage" class="anchor" aria-label="Permalink: Usage" href="#usage"><svg class="octicon octicon-link" viewbox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
  <div class="highlight highlight-source-sql notranslate position-relative overflow-auto" dir="auto" data-snippet-clipboard-copy-content=".load ./html0
  select html_extract('<p> Anakin <b>Skywalker</b> </p>', 'b');
  -- "<b>Skywalker</b>""><pre>.load .<span class="pl-k">/</span>html0
  <span class="pl-k">select</span> html_extract(<span class="pl-s"><span class="pl-pds">'</span><p> Anakin <b>Skywalker</b> </p><span class="pl-pds">'</span></span>, <span class="pl-s"><span class="pl-pds">'</span>b<span class="pl-pds">'</span></span>);
  <span class="pl-c"><span class="pl-c">--</span> "<b>Skywalker</b>"</span></pre></div>
  <p dir="auto"><code>sqlite-html</code> is similar to other HTML scraping tools like <a href="https://beautiful-soup-4.readthedocs.io/en/latest/" rel="nofollow">BeautifulSoup</a> (Python) or <a href="https://cheerio.js.org/" rel="nofollow">cheerio</a> (Node.js) or <a href="https://nokogiri.org/" rel="nofollow">nokogiri</a> (Ruby). You can use CSS selectors to extract individual elements or groups of elements to query data from HTML sources.</p>
  <p dir="auto">For example, here we find all <code>href</code> links in an <code>index.html</code> file.</p>
  <div class="highlight highlight-source-sql notranslate position-relative overflow-auto" dir="auto" data-snippet-clipboard-copy-content="select
    text as name,
    html_attribute_get(anchors, 'a', 'href') as href
  from html_each(readfile('index.html'), 'a') as anchors"><pre><span class="pl-k">select</span>
    <span class="pl-k">text</span> <span class="pl-k">as</span> name,
    html_attribute_get(anchors, <span class="pl-s"><span class="pl-pds">'</span>a<span class="pl-pds">'</span></span>, <span class="pl-s"><span class="pl-pds">'</span>href<span class="pl-pds">'</span></span>) <span class="pl-k">as</span> href
  <span class="pl-k">from</span> html_each(readfile(<span class="pl-s"><span class="pl-pds">'</span>index.html<span class="pl-pds">'</span></span>), <span class="pl-s"><span class="pl-pds">'</span>a<span class="pl-pds">'</span></span>) <span class="pl-k">as</span> anchors</pre></div>
  <p dir="auto">We can also safely generate HTML with <code>html_element</code>, modeled after React's <a href="https://reactjs.org/docs/react-api.html#createelement" rel="nofollow"><code>React.createElement</code></a>.</p>
  <div class="highlight highlight-source-sql notranslate position-relative overflow-auto" dir="auto" data-snippet-clipboard-copy-content="select html_element('p', null,
    'Luke, I am your',
    html_element('b', null, 'father'),
    '!',

    html_element('img', json_object(
      'src', 'https://images.dog.ceo/breeds/groenendael/n02105056_4600.jpg',
      'width', 200
    ))
  );

  -- "<p>Luke, I am your<b>father</b>!<img src="https://images.dog.ceo/breeds/groenendael/n02105056_4600.jpg" width="200.000000"/></p>""><pre><span class="pl-k">select</span> html_element(<span class="pl-s"><span class="pl-pds">'</span>p<span class="pl-pds">'</span></span>, <span class="pl-k">null</span>,
    <span class="pl-s"><span class="pl-pds">'</span>Luke, I am your<span class="pl-pds">'</span></span>,
    html_element(<span class="pl-s"><span class="pl-pds">'</span>b<span class="pl-pds">'</span></span>, <span class="pl-k">null</span>, <span class="pl-s"><span class="pl-pds">'</span>father<span class="pl-pds">'</span></span>),
    <span class="pl-s"><span class="pl-pds">'</span>!<span class="pl-pds">'</span></span>,

    html_element(<span class="pl-s"><span class="pl-pds">'</span>img<span class="pl-pds">'</span></span>, json_object(
      <span class="pl-s"><span class="pl-pds">'</span>src<span class="pl-pds">'</span></span>, <span class="pl-s"><span class="pl-pds">'</span>https://images.dog.ceo/breeds/groenendael/n02105056_4600.jpg<span class="pl-pds">'</span></span>,
      <span class="pl-s"><span class="pl-pds">'</span>width<span class="pl-pds">'</span></span>, <span class="pl-c1">200</span>
    ))
  );

  <span class="pl-c"><span class="pl-c">--</span> "<p>Luke, I am your<b>father</b>!<img src="https://images.dog.ceo/breeds/groenendael/n02105056_4600.jpg" width="200.000000"/></p>"</span></pre></div>
  <div class="markdown-heading" dir="auto"><h2 class="heading-element" dir="auto">Documentation</h2><a id="user-content-documentation" class="anchor" aria-label="Permalink: Documentation" href="#documentation"><svg class="octicon octicon-link" viewbox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
  <p dir="auto">See <a href="./docs.md"><code>docs.md</code></a> for a full API reference.</p>
  <div class="markdown-heading" dir="auto"><h2 class="heading-element" dir="auto">Installing</h2><a id="user-content-installing" class="anchor" aria-label="Permalink: Installing" href="#installing"><svg class="octicon octicon-link" viewbox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
  <markdown-accessiblity-table><table>
  <thead>
  <tr>
  <th>Language</th>
  <th>Install</th>
  <th></th>
  </tr>
  </thead>
  <tbody>
  <tr>
  <td>Python</td>
  <td><code>pip install sqlite-html</code></td>
  <td><a href="https://pypi.org/project/sqlite-html/" rel="nofollow"><img src="https://img.shields.io/pypi/v/sqlite-html.svg?color=blue&logo=python&logoColor=white" alt="PyPI" style="max-width: 100%;" data-camo-src="https://camo.githubusercontent.com/5bbe21917dbdaada3932b5bbae148e226d6590bc885b2a0d7d70f3f2ae7b97fa/68747470733a2f2f696d672e736869656c64732e696f2f707970692f762f73716c6974652d68746d6c2e7376673f636f6c6f723d626c7565266c6f676f3d707974686f6e266c6f676f436f6c6f723d7768697465"></a></td>
  </tr>
  <tr>
  <td>Datasette</td>
  <td><code>datasette install datasette-sqlite-html</code></td>
  <td><a href="https://datasette.io/plugins/datasette-sqlite-html" rel="nofollow"><img src="https://img.shields.io/pypi/v/datasette-sqlite-html.svg?color=B6B6D9&label=Datasette+plugin&logoColor=white&logo=python" alt="Datasette" style="max-width: 100%;" data-camo-src="https://camo.githubusercontent.com/475a3d56114d1ab8ce4df9480f77bca2f92928d6d110e64b71c4c98da44bed7c/68747470733a2f2f696d672e736869656c64732e696f2f707970692f762f6461746173657474652d73716c6974652d68746d6c2e7376673f636f6c6f723d423642364439266c6162656c3d4461746173657474652b706c7567696e266c6f676f436f6c6f723d7768697465266c6f676f3d707974686f6e"></a></td>
  </tr>
  <tr>
  <td>Node.js</td>
  <td><code>npm install sqlite-html</code></td>
  <td><a href="https://www.npmjs.com/package/sqlite-html" rel="nofollow"><img src="https://img.shields.io/npm/v/sqlite-html.svg?color=green&logo=nodedotjs&logoColor=white" alt="npm" style="max-width: 100%;" data-camo-src="https://camo.githubusercontent.com/0d423afd584009f1964caa4813809993a1729439d4ed957f4d66c60f030eb654/68747470733a2f2f696d672e736869656c64732e696f2f6e706d2f762f73716c6974652d68746d6c2e7376673f636f6c6f723d677265656e266c6f676f3d6e6f6465646f746a73266c6f676f436f6c6f723d7768697465"></a></td>
  </tr>
  <tr>
  <td>Deno</td>
  <td><a href="https://deno.land/x/sqlite_html" rel="nofollow"><code>deno.land/x/sqlite_html</code></a></td>
  <td><a href="https://deno.land/x/sqlite_html" rel="nofollow"><img src="https://img.shields.io/github/v/release/asg017/sqlite-html?color=fef8d2&include_prereleases&label=deno.land%2Fx&logo=deno" alt="deno.land/x release" style="max-width: 100%;" data-camo-src="https://camo.githubusercontent.com/03bbeb54197074fb591ca3203bdcf45c0f8e26521dc2119b2b6d34f8607cd1ae/68747470733a2f2f696d672e736869656c64732e696f2f6769746875622f762f72656c656173652f6173673031372f73716c6974652d68746d6c3f636f6c6f723d66656638643226696e636c7564655f70726572656c6561736573266c6162656c3d64656e6f2e6c616e6425324678266c6f676f3d64656e6f"></a></td>
  </tr>
  <tr>
  <td>Ruby</td>
  <td><code>gem install sqlite-html</code></td>
  <td><a target="_blank" rel="noopener noreferrer nofollow" href="https://camo.githubusercontent.com/f66aabfe986df4f3068ffddf1612973f7eeebc334364a6d82c4810f812730a6a/68747470733a2f2f696d672e736869656c64732e696f2f67656d2f762f73716c6974652d68746d6c3f636f6c6f723d726564266c6f676f3d7275627967656d73266c6f676f436f6c6f723d7768697465"><img src="https://img.shields.io/gem/v/sqlite-html?color=red&logo=rubygems&logoColor=white" alt="Gem" style="max-width: 100%;" data-camo-src="https://camo.githubusercontent.com/f66aabfe986df4f3068ffddf1612973f7eeebc334364a6d82c4810f812730a6a/68747470733a2f2f696d672e736869656c64732e696f2f67656d2f762f73716c6974652d68746d6c3f636f6c6f723d726564266c6f676f3d7275627967656d73266c6f676f436f6c6f723d7768697465"></a></td>
  </tr>
  <tr>
  <td>Github Release</td>
  <td></td>
  <td><a target="_blank" rel="noopener noreferrer nofollow" href="https://camo.githubusercontent.com/8eb70b59fed278743a6e0fb77c2541e3a43a68ea65a133b263f4a691d8004cd1/68747470733a2f2f696d672e736869656c64732e696f2f6769746875622f762f7461672f6173673031372f73716c6974652d68746d6c3f636f6c6f723d6c696768746772657926696e636c7564655f70726572656c6561736573266c6162656c3d4769746875622b72656c65617365266c6f676f3d676974687562"><img src="https://img.shields.io/github/v/tag/asg017/sqlite-html?color=lightgrey&include_prereleases&label=Github+release&logo=github" alt="GitHub tag (latest SemVer pre-release)" style="max-width: 100%;" data-camo-src="https://camo.githubusercontent.com/8eb70b59fed278743a6e0fb77c2541e3a43a68ea65a133b263f4a691d8004cd1/68747470733a2f2f696d672e736869656c64732e696f2f6769746875622f762f7461672f6173673031372f73716c6974652d68746d6c3f636f6c6f723d6c696768746772657926696e636c7564655f70726572656c6561736573266c6162656c3d4769746875622b72656c65617365266c6f676f3d676974687562"></a></td>
  </tr>
  </tbody>
  </table></markdown-accessiblity-table>

  <p dir="auto">The <a href="https://github.com/asg017/sqlite-html/releases">Releases page</a> contains pre-built binaries for Linux amd64, MacOS amd64 (no arm), and Windows.</p>
  <div class="markdown-heading" dir="auto"><h3 class="heading-element" dir="auto">As a loadable extension</h3><a id="user-content-as-a-loadable-extension" class="anchor" aria-label="Permalink: As a loadable extension" href="#as-a-loadable-extension"><svg class="octicon octicon-link" viewbox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
  <p dir="auto">If you want to use <code>sqlite-html</code> as a <a href="https://www.sqlite.org/loadext.html" rel="nofollow">Runtime-loadable extension</a>, Download the <code>html0.dylib</code> (for MacOS), <code>html0.so</code> (Linux), or <code>html0.dll</code> (Windows) file from a release and load it into your SQLite environment.</p>
  <blockquote>
  <p dir="auto"><strong>Note:</strong>
  The <code>0</code> in the filename (<code>html0.dylib</code>/ <code>html0.so</code>/<code>html0.dll</code>) denotes the major version of <code>sqlite-html</code>. Currently <code>sqlite-html</code> is pre v1, so expect breaking changes in future versions.</p>
  </blockquote>
  <p dir="auto">For example, if you are using the <a href="https://www.sqlite.org/cli.html" rel="nofollow">SQLite CLI</a>, you can load the library like so:</p>
  <div class="highlight highlight-source-sql notranslate position-relative overflow-auto" dir="auto" data-snippet-clipboard-copy-content=".load ./html0
  select html_version();
  -- v0.0.1"><pre>.load .<span class="pl-k">/</span>html0
  <span class="pl-k">select</span> html_version();
  <span class="pl-c"><span class="pl-c">--</span> v0.0.1</span></pre></div>
  <p dir="auto">Or in Python, using the builtin <a href="https://docs.python.org/3/library/sqlite3.html" rel="nofollow">sqlite3 module</a>:</p>
  <div class="highlight highlight-source-python notranslate position-relative overflow-auto" dir="auto" data-snippet-clipboard-copy-content="import sqlite3

  con = sqlite3.connect(":memory:")

  con.enable_load_extension(True)
  con.load_extension("./html0")

  print(con.execute("select html_version()").fetchone())
  # ('v0.0.1',)"><pre><span class="pl-k">import</span> <span class="pl-s1">sqlite3</span>

  <span class="pl-s1">con</span> <span class="pl-c1">=</span> <span class="pl-s1">sqlite3</span>.<span class="pl-c1">connect</span>(<span class="pl-s">":memory:"</span>)

  <span class="pl-s1">con</span>.<span class="pl-c1">enable_load_extension</span>(<span class="pl-c1">True</span>)
  <span class="pl-s1">con</span>.<span class="pl-c1">load_extension</span>(<span class="pl-s">"./html0"</span>)

  <span class="pl-en">print</span>(<span class="pl-s1">con</span>.<span class="pl-c1">execute</span>(<span class="pl-s">"select html_version()"</span>).<span class="pl-c1">fetchone</span>())
  <span class="pl-c"># ('v0.0.1',)</span></pre></div>
  <p dir="auto">Or in Node.js using <a href="https://github.com/WiseLibs/better-sqlite3">better-sqlite3</a>:</p>
  <div class="highlight highlight-source-js notranslate position-relative overflow-auto" dir="auto" data-snippet-clipboard-copy-content="const Database = require("better-sqlite3");
  const db = new Database(":memory:");

  db.loadExtension("./html0");

  console.log(db.prepare("select html_version()").get());
  // { 'html_version()': 'v0.0.1' }"><pre><span class="pl-k">const</span> <span class="pl-v">Database</span> <span class="pl-c1">=</span> <span class="pl-en">require</span><span class="pl-kos">(</span><span class="pl-s">"better-sqlite3"</span><span class="pl-kos">)</span><span class="pl-kos">;</span>
  <span class="pl-k">const</span> <span class="pl-s1">db</span> <span class="pl-c1">=</span> <span class="pl-k">new</span> <span class="pl-v">Database</span><span class="pl-kos">(</span><span class="pl-s">":memory:"</span><span class="pl-kos">)</span><span class="pl-kos">;</span>

  <span class="pl-s1">db</span><span class="pl-kos">.</span><span class="pl-en">loadExtension</span><span class="pl-kos">(</span><span class="pl-s">"./html0"</span><span class="pl-kos">)</span><span class="pl-kos">;</span>

  <span class="pl-smi">console</span><span class="pl-kos">.</span><span class="pl-en">log</span><span class="pl-kos">(</span><span class="pl-s1">db</span><span class="pl-kos">.</span><span class="pl-en">prepare</span><span class="pl-kos">(</span><span class="pl-s">"select html_version()"</span><span class="pl-kos">)</span><span class="pl-kos">.</span><span class="pl-en">get</span><span class="pl-kos">(</span><span class="pl-kos">)</span><span class="pl-kos">)</span><span class="pl-kos">;</span>
  <span class="pl-c">// { 'html_version()': 'v0.0.1' }</span></pre></div>
  <p dir="auto">Or with <a href="https://datasette.io/" rel="nofollow">Datasette</a>:</p>
  <div class="snippet-clipboard-content notranslate position-relative overflow-auto" data-snippet-clipboard-copy-content="datasette data.db --load-extension ./html0"><pre class="notranslate"><code>datasette data.db --load-extension ./html0
  </code></pre></div>
  <div class="markdown-heading" dir="auto"><h2 class="heading-element" dir="auto">See also</h2><a id="user-content-see-also" class="anchor" aria-label="Permalink: See also" href="#see-also"><svg class="octicon octicon-link" viewbox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
  <ul dir="auto">
  <li><a href="https://github.com/asg017/sqlite-http">sqlite-http</a>, for making HTTP requests in SQLite (pairs great with this tool)</li>
  <li><a href="https://github.com/mgdm/htmlq">htmlq</a>, for a similar but CLI-based HTML query tool using CSS selectors</li>
  <li><a href="https://github.com/riyaz-ali/sqlite">riyaz-ali/sqlite</a>, the brilliant Go library that this library depends on</li>
  <li><a href="https://github.com/nalgeon/sqlean">nalgeon/sqlean</a>, several pre-compiled handy SQLite functions, in C</li>
  </ul>
  </article></div>


node_id: R_kgDOGM0Ptw
name: sqlite-http
full_name: asg017/sqlite-http
html_url: https://github.com/asg017/sqlite-http
description: A SQLite extension for making HTTP requests purely in SQL
created_at: 2021-10-11T21:27:49Z
updated_at: 2025-03-25T14:10:00Z
pushed_at: 2025-02-09T16:15:00Z
language: Go
license: mit
topics: ["sqlite", "sqlite-extension"]
visibility: public
default_branch: main
permissions: {"admin": false, "maintain": false, "push": false, "triage": false, "pull": true}
readme:
  # sqlite-http

  A SQLite extension for making HTTP requests purely in SQL.

  - Create GET, POST, and other HTTP requests, like `curl`, `wget`, and `fetch`
  - Download response bodies, header, status codes, timing info
  - Set rate limits, timeouts

  ## Usage

  ```sql
  .load ./http0
  select http_get_body('https://text.npr.org/');
  /*
  <!DOCTYPE html>
  <html lang="en">
  <head>
    <title>NPR : National Public Radio</title>
    ....
  */
  ```

  Query for all custom headers in an endpoint.

  ```sql
  select name, value
  from http_headers_each(
    http_get_headers('https://api.github.com/')
  )
  where name like 'X-%';
  /*
  ┌────────────────────────┬────────────────────────────────────┐
  │          name          │               value                │
  ├────────────────────────┼────────────────────────────────────┤
  │ X-Ratelimit-Limit      │ 60                                 │
  │ X-Ratelimit-Used       │ 8                                  │
  │ X-Content-Type-Options │ nosniff                            │
  │ X-Github-Media-Type    │ github.v3; format=json             │
  │ X-Github-Request-Id    │ CCCA:5FDF:1014BC2:10965F9:62F3DE4E │
  │ X-Ratelimit-Remaining  │ 52                                 │
  │ X-Ratelimit-Resource   │ core                               │
  │ X-Frame-Options        │ deny                               │
  │ X-Ratelimit-Reset      │ 1660152798                         │
  │ X-Xss-Protection       │ 0                                  │
  └────────────────────────┴────────────────────────────────────┘
  */
  ```

  Scrape data from a JSON endpoint.

  ```sql
  select http_get_body('https://api.github.com/repos/sqlite/sqlite')
    ->> '$.description' as description;
  /*
  ┌───────────────────────────────────────────────┐
  │                  description                  │
  ├───────────────────────────────────────────────┤
  │ Official Git mirror of the SQLite source tree │
  └───────────────────────────────────────────────┘
  */
  ```

  Pass in specific headers in a request.

  ```sql
  select
    value
  from json_each(
    http_get_body(
      'https://api.github.com/issues',
      http_headers(
        'Authorization', 'token ghp_16C7e42F292c6912E7710c8'
      )
    )
  );

  ```

  ## Documentation

  See [`docs.md`](./docs.md) for a full API reference.

  ## Installing

  | Language       | Install                                                      |                                                                                                                                                                                             |
  | -------------- | ------------------------------------------------------------ | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
  | Python         | `pip install sqlite-http`                                    | [![PyPI](https://img.shields.io/pypi/v/sqlite-http.svg?color=blue&logo=python&logoColor=white)](https://pypi.org/project/sqlite-http/)                                                      |
  | Datasette      | `datasette install datasette-sqlite-http`                    | [![Datasette](https://img.shields.io/pypi/v/datasette-sqlite-http.svg?color=B6B6D9&label=Datasette+plugin&logoColor=white&logo=python)](https://datasette.io/plugins/datasette-sqlite-http) |
  | Node.js        | `npm install sqlite-http`                                    | [![npm](https://img.shields.io/npm/v/sqlite-http.svg?color=green&logo=nodedotjs&logoColor=white)](https://www.npmjs.com/package/sqlite-http)                                                |
  | Deno           | [`deno.land/x/sqlite_http`](https://deno.land/x/sqlite_http) | [![deno.land/x release](https://img.shields.io/github/v/release/asg017/sqlite-http?color=fef8d2&include_prereleases&label=deno.land%2Fx&logo=deno)](https://deno.land/x/sqlite_http)        |
  | Ruby           | `gem install sqlite-http`                                    | ![Gem](https://img.shields.io/gem/v/sqlite-http?color=red&logo=rubygems&logoColor=white)                                                                                                    |
  | Github Release |                                                              | ![GitHub tag (latest SemVer pre-release)](https://img.shields.io/github/v/tag/asg017/sqlite-http?color=lightgrey&include_prereleases&label=Github+release&logo=github)                      |

  <!--
  | Elixir         | [`hex.pm/packages/sqlite_http`](https://hex.pm/packages/sqlite_http) | [![Hex.pm](https://img.shields.io/hexpm/v/sqlite_http?color=purple&logo=elixir)](https://hex.pm/packages/sqlite_http)                                                                       |
  | Go             | `go get -u github.com/asg017/sqlite-http/bindings/go`               | [![Go Reference](https://pkg.go.dev/badge/github.com/asg017/sqlite-http/bindings/go.svg)](https://pkg.go.dev/github.com/asg017/sqlite-http/bindings/go)                                     |
  | Rust           | `cargo add sqlite-http`                                             | [![Crates.io](https://img.shields.io/crates/v/sqlite-http?logo=rust)](https://crates.io/crates/sqlite-http)                                                                                 |
  -->

  The [Releases page](https://github.com/asg017/sqlite-http/releases) contains pre-built binaries for Linux amd64, MacOS amd64 (no arm), and Windows.

  ### As a loadable extension

  If you want to use `sqlite-http` as a [Runtime-loadable extension](https://www.sqlite.org/loadext.html), Download the `http0.dylib` (for MacOS), `http0.so` (Linux), or `http0.dll` (Windows) file from a release and load it into your SQLite environment.

  > **Note:**
  > The `0` in the filename (`http0.dylib`/ `http0.so`/`http0.dll`) denotes the major version of `sqlite-http`. Currently `sqlite-http` is pre v1, so expect breaking changes in future versions.

  For example, if you are using the [SQLite CLI](https://www.sqlite.org/cli.html), you can load the library like so:

  ```sql
  .load ./http0
  select http_version();
  -- v0.0.1
  ```

  Or in Python, using the builtin [sqlite3 module](https://docs.python.org/3/library/sqlite3.html):

  ```python
  import sqlite3

  con = sqlite3.connect(":memory:")

  con.enable_load_extension(True)
  con.load_extension("./http0")

  print(con.execute("select http_version()").fetchone())
  # ('v0.0.1',)
  ```

  Or in Node.js using [better-sqlite3](https://github.com/WiseLibs/better-sqlite3):

  ```javascript
  const Database = require("better-sqlite3");
  const db = new Database(":memory:");

  db.loadExtension("./http0");

  console.log(db.prepare("select http_version()").get());
  // { 'http_version()': 'v0.0.1' }
  ```

  Or with [Datasette](https://datasette.io/), with the "no network" option to limit DDoS attacks:

  ```
  datasette data.db --load-extension ./http0-no-net
  ```

  ## See also

  - [sqlite-html](https://github.com/asg017/sqlite-html), for parsing and querying HTML using CSS selectors in SQLite (pairs great with this tool)
  - [pgsql-http](https://github.com/pramsey/pgsql-http), a similar yet very different HTTP library for POstgreSQL (didn't know about this before I started this, but interestingly enough came up with a very similar API)
  - [riyaz-ali/sqlite](https://github.com/riyaz-ali/sqlite), the brilliant Go library that this library depends on
  - [nalgeon/sqlean](https://github.com/nalgeon/sqlean), several pre-compiled handy SQLite functions, in C

readme_html:
  <div id="readme" class="md" data-path="README.md"><article class="markdown-body entry-content container-lg" itemprop="text"><div class="markdown-heading" dir="auto"><h1 class="heading-element" dir="auto">sqlite-http</h1><a id="user-content-sqlite-http" class="anchor" aria-label="Permalink: sqlite-http" href="#sqlite-http"><svg class="octicon octicon-link" viewbox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
  <p dir="auto">A SQLite extension for making HTTP requests purely in SQL.</p>
  <ul dir="auto">
  <li>Create GET, POST, and other HTTP requests, like <code>curl</code>, <code>wget</code>, and <code>fetch</code></li>
  <li>Download response bodies, header, status codes, timing info</li>
  <li>Set rate limits, timeouts</li>
  </ul>
  <div class="markdown-heading" dir="auto"><h2 class="heading-element" dir="auto">Usage</h2><a id="user-content-usage" class="anchor" aria-label="Permalink: Usage" href="#usage"><svg class="octicon octicon-link" viewbox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
  <div class="highlight highlight-source-sql notranslate position-relative overflow-auto" dir="auto" data-snippet-clipboard-copy-content=".load ./http0
  select http_get_body('https://text.npr.org/');
  /*
  <!DOCTYPE html>
  <html lang="en">
  <head>
    <title>NPR : National Public Radio</title>
    ....
  */"><pre>.load .<span class="pl-k">/</span>http0
  <span class="pl-k">select</span> http_get_body(<span class="pl-s"><span class="pl-pds">'</span>https://text.npr.org/<span class="pl-pds">'</span></span>);
  <span class="pl-c"><span class="pl-c">/*</span></span>
  <span class="pl-c"><!DOCTYPE html></span>
  <span class="pl-c"><html lang="en"></span>
  <span class="pl-c"><head></span>
  <span class="pl-c">  <title>NPR : National Public Radio</title></span>
  <span class="pl-c">  ....</span>
  <span class="pl-c"><span class="pl-c">*/</span></span></pre></div>
  <p dir="auto">Query for all custom headers in an endpoint.</p>
  <div class="highlight highlight-source-sql notranslate position-relative overflow-auto" dir="auto" data-snippet-clipboard-copy-content="select name, value
  from http_headers_each(
    http_get_headers('https://api.github.com/')
  )
  where name like 'X-%';
  /*
  ┌────────────────────────┬────────────────────────────────────┐
  │          name          │               value                │
  ├────────────────────────┼────────────────────────────────────┤
  │ X-Ratelimit-Limit      │ 60                                 │
  │ X-Ratelimit-Used       │ 8                                  │
  │ X-Content-Type-Options │ nosniff                            │
  │ X-Github-Media-Type    │ github.v3; format=json             │
  │ X-Github-Request-Id    │ CCCA:5FDF:1014BC2:10965F9:62F3DE4E │
  │ X-Ratelimit-Remaining  │ 52                                 │
  │ X-Ratelimit-Resource   │ core                               │
  │ X-Frame-Options        │ deny                               │
  │ X-Ratelimit-Reset      │ 1660152798                         │
  │ X-Xss-Protection       │ 0                                  │
  └────────────────────────┴────────────────────────────────────┘
  */"><pre><span class="pl-k">select</span> name, value
  <span class="pl-k">from</span> http_headers_each(
    http_get_headers(<span class="pl-s"><span class="pl-pds">'</span>https://api.github.com/<span class="pl-pds">'</span></span>)
  )
  <span class="pl-k">where</span> name <span class="pl-k">like</span> <span class="pl-s"><span class="pl-pds">'</span>X-%<span class="pl-pds">'</span></span>;
  <span class="pl-c"><span class="pl-c">/*</span></span>
  <span class="pl-c">┌────────────────────────┬────────────────────────────────────┐</span>
  <span class="pl-c">│          name          │               value                │</span>
  <span class="pl-c">├────────────────────────┼────────────────────────────────────┤</span>
  <span class="pl-c">│ X-Ratelimit-Limit      │ 60                                 │</span>
  <span class="pl-c">│ X-Ratelimit-Used       │ 8                                  │</span>
  <span class="pl-c">│ X-Content-Type-Options │ nosniff                            │</span>
  <span class="pl-c">│ X-Github-Media-Type    │ github.v3; format=json             │</span>
  <span class="pl-c">│ X-Github-Request-Id    │ CCCA:5FDF:1014BC2:10965F9:62F3DE4E │</span>
  <span class="pl-c">│ X-Ratelimit-Remaining  │ 52                                 │</span>
  <span class="pl-c">│ X-Ratelimit-Resource   │ core                               │</span>
  <span class="pl-c">│ X-Frame-Options        │ deny                               │</span>
  <span class="pl-c">│ X-Ratelimit-Reset      │ 1660152798                         │</span>
  <span class="pl-c">│ X-Xss-Protection       │ 0                                  │</span>
  <span class="pl-c">└────────────────────────┴────────────────────────────────────┘</span>
  <span class="pl-c"><span class="pl-c">*/</span></span></pre></div>
  <p dir="auto">Scrape data from a JSON endpoint.</p>
  <div class="highlight highlight-source-sql notranslate position-relative overflow-auto" dir="auto" data-snippet-clipboard-copy-content="select http_get_body('https://api.github.com/repos/sqlite/sqlite')
    ->> '$.description' as description;
  /*
  ┌───────────────────────────────────────────────┐
  │                  description                  │
  ├───────────────────────────────────────────────┤
  │ Official Git mirror of the SQLite source tree │
  └───────────────────────────────────────────────┘
  */"><pre><span class="pl-k">select</span> http_get_body(<span class="pl-s"><span class="pl-pds">'</span>https://api.github.com/repos/sqlite/sqlite<span class="pl-pds">'</span></span>)
    <span class="pl-k">-</span><span class="pl-k">>></span> <span class="pl-s"><span class="pl-pds">'</span>$.description<span class="pl-pds">'</span></span> <span class="pl-k">as</span> description;
  <span class="pl-c"><span class="pl-c">/*</span></span>
  <span class="pl-c">┌───────────────────────────────────────────────┐</span>
  <span class="pl-c">│                  description                  │</span>
  <span class="pl-c">├───────────────────────────────────────────────┤</span>
  <span class="pl-c">│ Official Git mirror of the SQLite source tree │</span>
  <span class="pl-c">└───────────────────────────────────────────────┘</span>
  <span class="pl-c"><span class="pl-c">*/</span></span></pre></div>
  <p dir="auto">Pass in specific headers in a request.</p>
  <div class="highlight highlight-source-sql notranslate position-relative overflow-auto" dir="auto" data-snippet-clipboard-copy-content="select
    value
  from json_each(
    http_get_body(
      'https://api.github.com/issues',
      http_headers(
        'Authorization', 'token ghp_16C7e42F292c6912E7710c8'
      )
    )
  );
  "><pre><span class="pl-k">select</span>
    value
  <span class="pl-k">from</span> json_each(
    http_get_body(
      <span class="pl-s"><span class="pl-pds">'</span>https://api.github.com/issues<span class="pl-pds">'</span></span>,
      http_headers(
        <span class="pl-s"><span class="pl-pds">'</span>Authorization<span class="pl-pds">'</span></span>, <span class="pl-s"><span class="pl-pds">'</span>token ghp_16C7e42F292c6912E7710c8<span class="pl-pds">'</span></span>
      )
    )
  );
  </pre></div>
  <div class="markdown-heading" dir="auto"><h2 class="heading-element" dir="auto">Documentation</h2><a id="user-content-documentation" class="anchor" aria-label="Permalink: Documentation" href="#documentation"><svg class="octicon octicon-link" viewbox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
  <p dir="auto">See <a href="./docs.md"><code>docs.md</code></a> for a full API reference.</p>
  <div class="markdown-heading" dir="auto"><h2 class="heading-element" dir="auto">Installing</h2><a id="user-content-installing" class="anchor" aria-label="Permalink: Installing" href="#installing"><svg class="octicon octicon-link" viewbox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
  <markdown-accessiblity-table><table>
  <thead>
  <tr>
  <th>Language</th>
  <th>Install</th>
  <th></th>
  </tr>
  </thead>
  <tbody>
  <tr>
  <td>Python</td>
  <td><code>pip install sqlite-http</code></td>
  <td><a href="https://pypi.org/project/sqlite-http/" rel="nofollow"><img src="https://img.shields.io/pypi/v/sqlite-http.svg?color=blue&logo=python&logoColor=white" alt="PyPI" style="max-width: 100%;" data-camo-src="https://camo.githubusercontent.com/80f9db9f195f8b1729ba0cfac0277f71da0b3970ab9af9b2c8d2bb6ba10235ff/68747470733a2f2f696d672e736869656c64732e696f2f707970692f762f73716c6974652d687474702e7376673f636f6c6f723d626c7565266c6f676f3d707974686f6e266c6f676f436f6c6f723d7768697465"></a></td>
  </tr>
  <tr>
  <td>Datasette</td>
  <td><code>datasette install datasette-sqlite-http</code></td>
  <td><a href="https://datasette.io/plugins/datasette-sqlite-http" rel="nofollow"><img src="https://img.shields.io/pypi/v/datasette-sqlite-http.svg?color=B6B6D9&label=Datasette+plugin&logoColor=white&logo=python" alt="Datasette" style="max-width: 100%;" data-camo-src="https://camo.githubusercontent.com/c1a3850c7ca1f74cb6cac2ce7dd6e69299257e69e27dbc88f38f6630b870b775/68747470733a2f2f696d672e736869656c64732e696f2f707970692f762f6461746173657474652d73716c6974652d687474702e7376673f636f6c6f723d423642364439266c6162656c3d4461746173657474652b706c7567696e266c6f676f436f6c6f723d7768697465266c6f676f3d707974686f6e"></a></td>
  </tr>
  <tr>
  <td>Node.js</td>
  <td><code>npm install sqlite-http</code></td>
  <td><a href="https://www.npmjs.com/package/sqlite-http" rel="nofollow"><img src="https://img.shields.io/npm/v/sqlite-http.svg?color=green&logo=nodedotjs&logoColor=white" alt="npm" style="max-width: 100%;" data-camo-src="https://camo.githubusercontent.com/fd12cf483587c71a8bfce211066c47dcd3c7eb893cb7e74bfd3020e23410cffb/68747470733a2f2f696d672e736869656c64732e696f2f6e706d2f762f73716c6974652d687474702e7376673f636f6c6f723d677265656e266c6f676f3d6e6f6465646f746a73266c6f676f436f6c6f723d7768697465"></a></td>
  </tr>
  <tr>
  <td>Deno</td>
  <td><a href="https://deno.land/x/sqlite_http" rel="nofollow"><code>deno.land/x/sqlite_http</code></a></td>
  <td><a href="https://deno.land/x/sqlite_http" rel="nofollow"><img src="https://img.shields.io/github/v/release/asg017/sqlite-http?color=fef8d2&include_prereleases&label=deno.land%2Fx&logo=deno" alt="deno.land/x release" style="max-width: 100%;" data-camo-src="https://camo.githubusercontent.com/b4d74d18eede14a8e100ce67c264a54781a3ecfc82cdea002eda046591cdbc83/68747470733a2f2f696d672e736869656c64732e696f2f6769746875622f762f72656c656173652f6173673031372f73716c6974652d687474703f636f6c6f723d66656638643226696e636c7564655f70726572656c6561736573266c6162656c3d64656e6f2e6c616e6425324678266c6f676f3d64656e6f"></a></td>
  </tr>
  <tr>
  <td>Ruby</td>
  <td><code>gem install sqlite-http</code></td>
  <td><a target="_blank" rel="noopener noreferrer nofollow" href="https://camo.githubusercontent.com/1d03076d6e2c822d32b3873625fa0180184f03a46fb235a0b77308ca9872de28/68747470733a2f2f696d672e736869656c64732e696f2f67656d2f762f73716c6974652d687474703f636f6c6f723d726564266c6f676f3d7275627967656d73266c6f676f436f6c6f723d7768697465"><img src="https://img.shields.io/gem/v/sqlite-http?color=red&logo=rubygems&logoColor=white" alt="Gem" style="max-width: 100%;" data-camo-src="https://camo.githubusercontent.com/1d03076d6e2c822d32b3873625fa0180184f03a46fb235a0b77308ca9872de28/68747470733a2f2f696d672e736869656c64732e696f2f67656d2f762f73716c6974652d687474703f636f6c6f723d726564266c6f676f3d7275627967656d73266c6f676f436f6c6f723d7768697465"></a></td>
  </tr>
  <tr>
  <td>Github Release</td>
  <td></td>
  <td><a target="_blank" rel="noopener noreferrer nofollow" href="https://camo.githubusercontent.com/3173cfd78cec02db597cff2f78460d8cca0d5f66446771ca77267acc70c7a1f7/68747470733a2f2f696d672e736869656c64732e696f2f6769746875622f762f7461672f6173673031372f73716c6974652d687474703f636f6c6f723d6c696768746772657926696e636c7564655f70726572656c6561736573266c6162656c3d4769746875622b72656c65617365266c6f676f3d676974687562"><img src="https://img.shields.io/github/v/tag/asg017/sqlite-http?color=lightgrey&include_prereleases&label=Github+release&logo=github" alt="GitHub tag (latest SemVer pre-release)" style="max-width: 100%;" data-camo-src="https://camo.githubusercontent.com/3173cfd78cec02db597cff2f78460d8cca0d5f66446771ca77267acc70c7a1f7/68747470733a2f2f696d672e736869656c64732e696f2f6769746875622f762f7461672f6173673031372f73716c6974652d687474703f636f6c6f723d6c696768746772657926696e636c7564655f70726572656c6561736573266c6162656c3d4769746875622b72656c65617365266c6f676f3d676974687562"></a></td>
  </tr>
  </tbody>
  </table></markdown-accessiblity-table>

  <p dir="auto">The <a href="https://github.com/asg017/sqlite-http/releases">Releases page</a> contains pre-built binaries for Linux amd64, MacOS amd64 (no arm), and Windows.</p>
  <div class="markdown-heading" dir="auto"><h3 class="heading-element" dir="auto">As a loadable extension</h3><a id="user-content-as-a-loadable-extension" class="anchor" aria-label="Permalink: As a loadable extension" href="#as-a-loadable-extension"><svg class="octicon octicon-link" viewbox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
  <p dir="auto">If you want to use <code>sqlite-http</code> as a <a href="https://www.sqlite.org/loadext.html" rel="nofollow">Runtime-loadable extension</a>, Download the <code>http0.dylib</code> (for MacOS), <code>http0.so</code> (Linux), or <code>http0.dll</code> (Windows) file from a release and load it into your SQLite environment.</p>
  <blockquote>
  <p dir="auto"><strong>Note:</strong>
  The <code>0</code> in the filename (<code>http0.dylib</code>/ <code>http0.so</code>/<code>http0.dll</code>) denotes the major version of <code>sqlite-http</code>. Currently <code>sqlite-http</code> is pre v1, so expect breaking changes in future versions.</p>
  </blockquote>
  <p dir="auto">For example, if you are using the <a href="https://www.sqlite.org/cli.html" rel="nofollow">SQLite CLI</a>, you can load the library like so:</p>
  <div class="highlight highlight-source-sql notranslate position-relative overflow-auto" dir="auto" data-snippet-clipboard-copy-content=".load ./http0
  select http_version();
  -- v0.0.1"><pre>.load .<span class="pl-k">/</span>http0
  <span class="pl-k">select</span> http_version();
  <span class="pl-c"><span class="pl-c">--</span> v0.0.1</span></pre></div>
  <p dir="auto">Or in Python, using the builtin <a href="https://docs.python.org/3/library/sqlite3.html" rel="nofollow">sqlite3 module</a>:</p>
  <div class="highlight highlight-source-python notranslate position-relative overflow-auto" dir="auto" data-snippet-clipboard-copy-content="import sqlite3

  con = sqlite3.connect(":memory:")

  con.enable_load_extension(True)
  con.load_extension("./http0")

  print(con.execute("select http_version()").fetchone())
  # ('v0.0.1',)"><pre><span class="pl-k">import</span> <span class="pl-s1">sqlite3</span>

  <span class="pl-s1">con</span> <span class="pl-c1">=</span> <span class="pl-s1">sqlite3</span>.<span class="pl-c1">connect</span>(<span class="pl-s">":memory:"</span>)

  <span class="pl-s1">con</span>.<span class="pl-c1">enable_load_extension</span>(<span class="pl-c1">True</span>)
  <span class="pl-s1">con</span>.<span class="pl-c1">load_extension</span>(<span class="pl-s">"./http0"</span>)

  <span class="pl-en">print</span>(<span class="pl-s1">con</span>.<span class="pl-c1">execute</span>(<span class="pl-s">"select http_version()"</span>).<span class="pl-c1">fetchone</span>())
  <span class="pl-c"># ('v0.0.1',)</span></pre></div>
  <p dir="auto">Or in Node.js using <a href="https://github.com/WiseLibs/better-sqlite3">better-sqlite3</a>:</p>
  <div class="highlight highlight-source-js notranslate position-relative overflow-auto" dir="auto" data-snippet-clipboard-copy-content="const Database = require("better-sqlite3");
  const db = new Database(":memory:");

  db.loadExtension("./http0");

  console.log(db.prepare("select http_version()").get());
  // { 'http_version()': 'v0.0.1' }"><pre><span class="pl-k">const</span> <span class="pl-v">Database</span> <span class="pl-c1">=</span> <span class="pl-en">require</span><span class="pl-kos">(</span><span class="pl-s">"better-sqlite3"</span><span class="pl-kos">)</span><span class="pl-kos">;</span>
  <span class="pl-k">const</span> <span class="pl-s1">db</span> <span class="pl-c1">=</span> <span class="pl-k">new</span> <span class="pl-v">Database</span><span class="pl-kos">(</span><span class="pl-s">":memory:"</span><span class="pl-kos">)</span><span class="pl-kos">;</span>

  <span class="pl-s1">db</span><span class="pl-kos">.</span><span class="pl-en">loadExtension</span><span class="pl-kos">(</span><span class="pl-s">"./http0"</span><span class="pl-kos">)</span><span class="pl-kos">;</span>

  <span class="pl-smi">console</span><span class="pl-kos">.</span><span class="pl-en">log</span><span class="pl-kos">(</span><span class="pl-s1">db</span><span class="pl-kos">.</span><span class="pl-en">prepare</span><span class="pl-kos">(</span><span class="pl-s">"select http_version()"</span><span class="pl-kos">)</span><span class="pl-kos">.</span><span class="pl-en">get</span><span class="pl-kos">(</span><span class="pl-kos">)</span><span class="pl-kos">)</span><span class="pl-kos">;</span>
  <span class="pl-c">// { 'http_version()': 'v0.0.1' }</span></pre></div>
  <p dir="auto">Or with <a href="https://datasette.io/" rel="nofollow">Datasette</a>, with the "no network" option to limit DDoS attacks:</p>
  <div class="snippet-clipboard-content notranslate position-relative overflow-auto" data-snippet-clipboard-copy-content="datasette data.db --load-extension ./http0-no-net"><pre class="notranslate"><code>datasette data.db --load-extension ./http0-no-net
  </code></pre></div>
  <div class="markdown-heading" dir="auto"><h2 class="heading-element" dir="auto">See also</h2><a id="user-content-see-also" class="anchor" aria-label="Permalink: See also" href="#see-also"><svg class="octicon octicon-link" viewbox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
  <ul dir="auto">
  <li><a href="https://github.com/asg017/sqlite-html">sqlite-html</a>, for parsing and querying HTML using CSS selectors in SQLite (pairs great with this tool)</li>
  <li><a href="https://github.com/pramsey/pgsql-http">pgsql-http</a>, a similar yet very different HTTP library for POstgreSQL (didn't know about this before I started this, but interestingly enough came up with a very similar API)</li>
  <li><a href="https://github.com/riyaz-ali/sqlite">riyaz-ali/sqlite</a>, the brilliant Go library that this library depends on</li>
  <li><a href="https://github.com/nalgeon/sqlean">nalgeon/sqlean</a>, several pre-compiled handy SQLite functions, in C</li>
  </ul>
  </article></div>


node_id: R_kgDOGT91rg
name: datasette-jupyterlite
full_name: simonw/datasette-jupyterlite
html_url: https://github.com/simonw/datasette-jupyterlite
description: JupyterLite as a Datasette plugin
created_at: 2021-11-01T19:22:51Z
updated_at: 2022-09-19T20:29:54Z
pushed_at: 2021-11-05T05:12:33Z
language: Python
topics: []
visibility: public
default_branch: main
permissions: {"admin": false, "maintain": false, "push": false, "triage": false, "pull": true}
readme:
  # datasette-jupyterlite

  [![PyPI](https://img.shields.io/pypi/v/datasette-jupyterlite.svg)](https://pypi.org/project/datasette-jupyterlite/)
  [![Changelog](https://img.shields.io/github/v/release/simonw/datasette-jupyterlite?include_prereleases&label=changelog)](https://github.com/simonw/datasette-jupyterlite/releases)
  [![Tests](https://github.com/simonw/datasette-jupyterlite/workflows/Test/badge.svg)](https://github.com/simonw/datasette-jupyterlite/actions?query=workflow%3ATest)
  [![License](https://img.shields.io/badge/license-Apache%202.0-blue.svg)](https://github.com/simonw/datasette-jupyterlite/blob/main/LICENSE)

  [JupyterLite](https://jupyterlite.readthedocs.io/en/latest/) as a Datasette plugin

  ## Installation

  Install this plugin in the same environment as Datasette.

      $ datasette install datasette-jupyterlite

  ## Demo

  You can try out a demo of the plugin here: https://latest-with-plugins.datasette.io/jupyterlite/

  Run this example code in a Pyolite notebook to pull all of the data from the [github/stars](https://latest-with-plugins.datasette.io/github/stars) table into a Pandas DataFrame:

  ```python
  import pandas, pyodide

  df = pandas.read_csv(pyodide.open_url(
    "https://latest-with-plugins.datasette.io/github/stars.csv?_labels=on&_stream=on&_size=max")
  )
  ```

  ## Usage

  Once installed, visit `/jupyterlite/` to access JupyterLite served from your Datasette instance.

  ## Development

  To set up this plugin locally, first checkout the code. Then create a new virtual environment:

      cd datasette-jupyterlite
      python3 -mvenv venv
      source venv/bin/activate

  Or if you are using `pipenv`:

      pipenv shell

  Now install the dependencies and test dependencies:

      pip install -e '.[test]'

  To run the tests:

      pytest

readme_html:
  <div id="readme" class="md" data-path="README.md"><article class="markdown-body entry-content container-lg" itemprop="text"><div class="markdown-heading" dir="auto"><h1 class="heading-element" dir="auto">datasette-jupyterlite</h1><a id="user-content-datasette-jupyterlite" class="anchor" aria-label="Permalink: datasette-jupyterlite" href="#datasette-jupyterlite"><svg class="octicon octicon-link" viewbox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
  <p dir="auto"><a href="https://pypi.org/project/datasette-jupyterlite/" rel="nofollow"><img src="https://img.shields.io/pypi/v/datasette-jupyterlite.svg" alt="PyPI" style="max-width: 100%;" data-camo-src="https://camo.githubusercontent.com/07cb4cfb122c9ba85bbf81ee1d0d70bbadb5681d0743edb96f230f4c58c5aa85/68747470733a2f2f696d672e736869656c64732e696f2f707970692f762f6461746173657474652d6a7570797465726c6974652e737667"></a>
  <a href="https://github.com/simonw/datasette-jupyterlite/releases"><img src="https://img.shields.io/github/v/release/simonw/datasette-jupyterlite?include_prereleases&label=changelog" alt="Changelog" style="max-width: 100%;" data-camo-src="https://camo.githubusercontent.com/60ef418abeca50bc0aa23c2c753bda4f6f7039a57420e29a46e80c2d34b63b1b/68747470733a2f2f696d672e736869656c64732e696f2f6769746875622f762f72656c656173652f73696d6f6e772f6461746173657474652d6a7570797465726c6974653f696e636c7564655f70726572656c6561736573266c6162656c3d6368616e67656c6f67"></a>
  <a href="https://github.com/simonw/datasette-jupyterlite/actions?query=workflow%3ATest"><img src="https://github.com/simonw/datasette-jupyterlite/workflows/Test/badge.svg" alt="Tests" style="max-width: 100%;"></a>
  <a href="https://github.com/simonw/datasette-jupyterlite/blob/main/LICENSE"><img src="https://img.shields.io/badge/license-Apache%202.0-blue.svg" alt="License" style="max-width: 100%;" data-camo-src="https://camo.githubusercontent.com/c355f200ea90fddaa407b6eaab303663a669248ea3ca7b1fcf77dbe04ff5f48c/68747470733a2f2f696d672e736869656c64732e696f2f62616467652f6c6963656e73652d417061636865253230322e302d626c75652e737667"></a></p>
  <p dir="auto"><a href="https://jupyterlite.readthedocs.io/en/latest/" rel="nofollow">JupyterLite</a> as a Datasette plugin</p>
  <div class="markdown-heading" dir="auto"><h2 class="heading-element" dir="auto">Installation</h2><a id="user-content-installation" class="anchor" aria-label="Permalink: Installation" href="#installation"><svg class="octicon octicon-link" viewbox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
  <p dir="auto">Install this plugin in the same environment as Datasette.</p>
  <div class="snippet-clipboard-content notranslate position-relative overflow-auto" data-snippet-clipboard-copy-content="$ datasette install datasette-jupyterlite"><pre class="notranslate"><code>$ datasette install datasette-jupyterlite
  </code></pre></div>
  <div class="markdown-heading" dir="auto"><h2 class="heading-element" dir="auto">Demo</h2><a id="user-content-demo" class="anchor" aria-label="Permalink: Demo" href="#demo"><svg class="octicon octicon-link" viewbox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
  <p dir="auto">You can try out a demo of the plugin here: <a href="https://latest-with-plugins.datasette.io/jupyterlite/" rel="nofollow">https://latest-with-plugins.datasette.io/jupyterlite/</a></p>
  <p dir="auto">Run this example code in a Pyolite notebook to pull all of the data from the <a href="https://latest-with-plugins.datasette.io/github/stars" rel="nofollow">github/stars</a> table into a Pandas DataFrame:</p>
  <div class="highlight highlight-source-python notranslate position-relative overflow-auto" dir="auto" data-snippet-clipboard-copy-content="import pandas, pyodide

  df = pandas.read_csv(pyodide.open_url(
    "https://latest-with-plugins.datasette.io/github/stars.csv?_labels=on&_stream=on&_size=max")
  )"><pre><span class="pl-k">import</span> <span class="pl-s1">pandas</span>, <span class="pl-s1">pyodide</span>

  <span class="pl-s1">df</span> <span class="pl-c1">=</span> <span class="pl-s1">pandas</span>.<span class="pl-c1">read_csv</span>(<span class="pl-s1">pyodide</span>.<span class="pl-c1">open_url</span>(
    <span class="pl-s">"https://latest-with-plugins.datasette.io/github/stars.csv?_labels=on&_stream=on&_size=max"</span>)
  )</pre></div>
  <div class="markdown-heading" dir="auto"><h2 class="heading-element" dir="auto">Usage</h2><a id="user-content-usage" class="anchor" aria-label="Permalink: Usage" href="#usage"><svg class="octicon octicon-link" viewbox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
  <p dir="auto">Once installed, visit <code>/jupyterlite/</code> to access JupyterLite served from your Datasette instance.</p>
  <div class="markdown-heading" dir="auto"><h2 class="heading-element" dir="auto">Development</h2><a id="user-content-development" class="anchor" aria-label="Permalink: Development" href="#development"><svg class="octicon octicon-link" viewbox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
  <p dir="auto">To set up this plugin locally, first checkout the code. Then create a new virtual environment:</p>
  <div class="snippet-clipboard-content notranslate position-relative overflow-auto" data-snippet-clipboard-copy-content="cd datasette-jupyterlite
  python3 -mvenv venv
  source venv/bin/activate"><pre class="notranslate"><code>cd datasette-jupyterlite
  python3 -mvenv venv
  source venv/bin/activate
  </code></pre></div>
  <p dir="auto">Or if you are using <code>pipenv</code>:</p>
  <div class="snippet-clipboard-content notranslate position-relative overflow-auto" data-snippet-clipboard-copy-content="pipenv shell"><pre class="notranslate"><code>pipenv shell
  </code></pre></div>
  <p dir="auto">Now install the dependencies and test dependencies:</p>
  <div class="snippet-clipboard-content notranslate position-relative overflow-auto" data-snippet-clipboard-copy-content="pip install -e '.[test]'"><pre class="notranslate"><code>pip install -e '.[test]'
  </code></pre></div>
  <p dir="auto">To run the tests:</p>
  <div class="snippet-clipboard-content notranslate position-relative overflow-auto" data-snippet-clipboard-copy-content="pytest"><pre class="notranslate"><code>pytest
  </code></pre></div>
  </article></div>


node_id: R_kgDOGUV9ig
name: s3-credentials
full_name: simonw/s3-credentials
html_url: https://github.com/simonw/s3-credentials
description: A tool for creating credentials for accessing S3 buckets
created_at: 2021-11-02T20:09:50Z
updated_at: 2026-04-07T21:04:59Z
pushed_at: 2025-12-21T18:12:16Z
homepage: https://s3-credentials.readthedocs.io
language: Python
license: apache-2.0
topics: ["aws", "boto3", "s3"]
visibility: public
default_branch: main
permissions: {"admin": false, "maintain": false, "push": false, "triage": false, "pull": false}
pull_request_creation_policy: all
readme:
  # s3-credentials

  [![PyPI](https://img.shields.io/pypi/v/s3-credentials.svg)](https://pypi.org/project/s3-credentials/)
  [![Changelog](https://img.shields.io/github/v/release/simonw/s3-credentials?include_prereleases&label=changelog)](https://github.com/simonw/s3-credentials/releases)
  [![Tests](https://github.com/simonw/s3-credentials/workflows/Test/badge.svg)](https://github.com/simonw/s3-credentials/actions?query=workflow%3ATest)
  [![Documentation Status](https://readthedocs.org/projects/s3-credentials/badge/?version=latest)](https://s3-credentials.readthedocs.org/)
  [![License](https://img.shields.io/badge/license-Apache%202.0-blue.svg)](https://github.com/simonw/s3-credentials/blob/master/LICENSE)

  A tool for creating credentials for accessing S3 buckets

  For project background, see [s3-credentials: a tool for creating credentials for S3 buckets](https://simonwillison.net/2021/Nov/3/s3-credentials/) on my blog.

  ## Installation

      pip install s3-credentials

  ## Basic usage

  To create a new S3 bucket and output credentials that can be used with only that bucket:
  ```
  % s3-credentials create my-new-s3-bucket --create-bucket
  Created bucket:  my-new-s3-bucket
  Created user: s3.read-write.my-new-s3-bucket with permissions boundary: arn:aws:iam::aws:policy/AmazonS3FullAccess
  Attached policy s3.read-write.my-new-s3-bucket to user s3.read-write.my-new-s3-bucket
  Created access key for user: s3.read-write.my-new-s3-bucket
  {
      "UserName": "s3.read-write.my-new-s3-bucket",
      "AccessKeyId": "AKIAWXFXAIOZOYLZAEW5",
      "Status": "Active",
      "SecretAccessKey": "...",
      "CreateDate": "2021-11-03 01:38:24+00:00"
  }
  ```
  The tool can do a lot more than this. See the [documentation](https://s3-credentials.readthedocs.io/) for details.

  ## Documentation

  - [Full documentation](https://s3-credentials.readthedocs.io/)
  - [Command help reference](https://s3-credentials.readthedocs.io/en/stable/help.html)
  - [Release notes](https://github.com/simonw/s3-credentials/releases)

readme_html:
  <div id="readme" class="md" data-path="README.md"><article class="markdown-body entry-content container-lg" itemprop="text"><div class="markdown-heading" dir="auto"><h1 class="heading-element" dir="auto">s3-credentials</h1><a id="user-content-s3-credentials" class="anchor" aria-label="Permalink: s3-credentials" href="#s3-credentials"><svg class="octicon octicon-link" viewbox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
  <p dir="auto"><a href="https://pypi.org/project/s3-credentials/" rel="nofollow"><img src="https://img.shields.io/pypi/v/s3-credentials.svg" alt="PyPI" style="max-width: 100%;" data-camo-src="https://camo.githubusercontent.com/933e1b40a91c158019e3112311e5c9052102d594f88f03d6d026a1a253d34742/68747470733a2f2f696d672e736869656c64732e696f2f707970692f762f73332d63726564656e7469616c732e737667"></a>
  <a href="https://github.com/simonw/s3-credentials/releases"><img src="https://img.shields.io/github/v/release/simonw/s3-credentials?include_prereleases&label=changelog" alt="Changelog" style="max-width: 100%;" data-camo-src="https://camo.githubusercontent.com/e9ab06120ab521f792d4872853ee311ceb051c6b2bb84f77fb49bd156d7b465b/68747470733a2f2f696d672e736869656c64732e696f2f6769746875622f762f72656c656173652f73696d6f6e772f73332d63726564656e7469616c733f696e636c7564655f70726572656c6561736573266c6162656c3d6368616e67656c6f67"></a>
  <a href="https://github.com/simonw/s3-credentials/actions?query=workflow%3ATest"><img src="https://github.com/simonw/s3-credentials/workflows/Test/badge.svg" alt="Tests" style="max-width: 100%;"></a>
  <a href="https://s3-credentials.readthedocs.org/" rel="nofollow"><img src="https://readthedocs.org/projects/s3-credentials/badge/?version=latest" alt="Documentation Status" style="max-width: 100%;" data-camo-src="https://camo.githubusercontent.com/6c8a57c3708d11b14213753e1cdf847eb70369c84adff5cfd1135b45a45a28c5/68747470733a2f2f72656164746865646f63732e6f72672f70726f6a656374732f73332d63726564656e7469616c732f62616467652f3f76657273696f6e3d6c6174657374"></a>
  <a href="https://github.com/simonw/s3-credentials/blob/master/LICENSE"><img src="https://img.shields.io/badge/license-Apache%202.0-blue.svg" alt="License" style="max-width: 100%;" data-camo-src="https://camo.githubusercontent.com/c355f200ea90fddaa407b6eaab303663a669248ea3ca7b1fcf77dbe04ff5f48c/68747470733a2f2f696d672e736869656c64732e696f2f62616467652f6c6963656e73652d417061636865253230322e302d626c75652e737667"></a></p>
  <p dir="auto">A tool for creating credentials for accessing S3 buckets</p>
  <p dir="auto">For project background, see <a href="https://simonwillison.net/2021/Nov/3/s3-credentials/" rel="nofollow">s3-credentials: a tool for creating credentials for S3 buckets</a> on my blog.</p>
  <div class="markdown-heading" dir="auto"><h2 class="heading-element" dir="auto">Installation</h2><a id="user-content-installation" class="anchor" aria-label="Permalink: Installation" href="#installation"><svg class="octicon octicon-link" viewbox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
  <div class="snippet-clipboard-content notranslate position-relative overflow-auto" data-snippet-clipboard-copy-content="pip install s3-credentials"><pre class="notranslate"><code>pip install s3-credentials
  </code></pre></div>
  <div class="markdown-heading" dir="auto"><h2 class="heading-element" dir="auto">Basic usage</h2><a id="user-content-basic-usage" class="anchor" aria-label="Permalink: Basic usage" href="#basic-usage"><svg class="octicon octicon-link" viewbox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
  <p dir="auto">To create a new S3 bucket and output credentials that can be used with only that bucket:</p>
  <div class="snippet-clipboard-content notranslate position-relative overflow-auto" data-snippet-clipboard-copy-content="% s3-credentials create my-new-s3-bucket --create-bucket
  Created bucket:  my-new-s3-bucket
  Created user: s3.read-write.my-new-s3-bucket with permissions boundary: arn:aws:iam::aws:policy/AmazonS3FullAccess
  Attached policy s3.read-write.my-new-s3-bucket to user s3.read-write.my-new-s3-bucket
  Created access key for user: s3.read-write.my-new-s3-bucket
  {
      "UserName": "s3.read-write.my-new-s3-bucket",
      "AccessKeyId": "AKIAWXFXAIOZOYLZAEW5",
      "Status": "Active",
      "SecretAccessKey": "...",
      "CreateDate": "2021-11-03 01:38:24+00:00"
  }"><pre class="notranslate"><code>% s3-credentials create my-new-s3-bucket --create-bucket
  Created bucket:  my-new-s3-bucket
  Created user: s3.read-write.my-new-s3-bucket with permissions boundary: arn:aws:iam::aws:policy/AmazonS3FullAccess
  Attached policy s3.read-write.my-new-s3-bucket to user s3.read-write.my-new-s3-bucket
  Created access key for user: s3.read-write.my-new-s3-bucket
  {
      "UserName": "s3.read-write.my-new-s3-bucket",
      "AccessKeyId": "AKIAWXFXAIOZOYLZAEW5",
      "Status": "Active",
      "SecretAccessKey": "...",
      "CreateDate": "2021-11-03 01:38:24+00:00"
  }
  </code></pre></div>
  <p dir="auto">The tool can do a lot more than this. See the <a href="https://s3-credentials.readthedocs.io/" rel="nofollow">documentation</a> for details.</p>
  <div class="markdown-heading" dir="auto"><h2 class="heading-element" dir="auto">Documentation</h2><a id="user-content-documentation" class="anchor" aria-label="Permalink: Documentation" href="#documentation"><svg class="octicon octicon-link" viewbox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
  <ul dir="auto">
  <li><a href="https://s3-credentials.readthedocs.io/" rel="nofollow">Full documentation</a></li>
  <li><a href="https://s3-credentials.readthedocs.io/en/stable/help.html" rel="nofollow">Command help reference</a></li>
  <li><a href="https://github.com/simonw/s3-credentials/releases">Release notes</a></li>
  </ul>
  </article></div>


node_id: R_kgDOGXV4Ig
name: git-history
full_name: simonw/git-history
html_url: https://github.com/simonw/git-history
description: Tools for analyzing Git history using SQLite
created_at: 2021-11-11T20:07:06Z
updated_at: 2026-02-16T09:16:14Z
pushed_at: 2025-12-21T20:22:54Z
language: Python
license: apache-2.0
topics: ["git", "sqlite"]
visibility: public
default_branch: main
permissions: {"admin": false, "maintain": false, "push": false, "triage": false, "pull": false}
pull_request_creation_policy: all
template_repository:
  {"id": 401177473, "node_id": "MDEwOlJlcG9zaXRvcnk0MDExNzc0NzM=", "name": "click-app-template-repository", "full_name": "simonw/click-app-template-repository", "private": false, "owner": {"login": "simonw", "id": 9599, "node_id": "MDQ6VXNlcjk1OTk=", "avatar_url": "https://avatars.githubusercontent.com/u/9599?v=4", "gravatar_id": "", "url": "https://api.github.com/users/simonw", "html_url": "https://github.com/simonw", "followers_url": "https://api.github.com/users/simonw/followers", "following_url": "https://api.github.com/users/simonw/following{/other_user}", "gists_url": "https://api.github.com/users/simonw/gists{/gist_id}", "starred_url": "https://api.github.com/users/simonw/starred{/owner}{/repo}", "subscriptions_url": "https://api.github.com/users/simonw/subscriptions", "organizations_url": "https://api.github.com/users/simonw/orgs", "repos_url": "https://api.github.com/users/simonw/repos", "events_url": "https://api.github.com/users/simonw/events{/privacy}", "received_events_url": "https://api.github.com/users/simonw/received_events", "type": "User", "user_view_type": "public", "site_admin": false}, "html_url": "https://github.com/simonw/click-app-template-repository", "description": "GitHub template repository for creating new Python Click CLI tools, using the simonw/click-app cookiecutter template", "fork": false, "url": "https://api.github.com/repos/simonw/click-app-template-repository", "forks_url": "https://api.github.com/repos/simonw/click-app-template-repository/forks", "keys_url": "https://api.github.com/repos/simonw/click-app-template-repository/keys{/key_id}", "collaborators_url": "https://api.github.com/repos/simonw/click-app-template-repository/collaborators{/collaborator}", "teams_url": "https://api.github.com/repos/simonw/click-app-template-repository/teams", "hooks_url": "https://api.github.com/repos/simonw/click-app-template-repository/hooks", "issue_events_url": "https://api.github.com/repos/simonw/click-app-template-repository/issues/events{/number}", "events_url": "https://api.github.com/repos/simonw/click-app-template-repository/events", "assignees_url": "https://api.github.com/repos/simonw/click-app-template-repository/assignees{/user}", "branches_url": "https://api.github.com/repos/simonw/click-app-template-repository/branches{/branch}", "tags_url": "https://api.github.com/repos/simonw/click-app-template-repository/tags", "blobs_url": "https://api.github.com/repos/simonw/click-app-template-repository/git/blobs{/sha}", "git_tags_url": "https://api.github.com/repos/simonw/click-app-template-repository/git/tags{/sha}", "git_refs_url": "https://api.github.com/repos/simonw/click-app-template-repository/git/refs{/sha}", "trees_url": "https://api.github.com/repos/simonw/click-app-template-repository/git/trees{/sha}", "statuses_url": "https://api.github.com/repos/simonw/click-app-template-repository/statuses/{sha}", "languages_url": "https://api.github.com/repos/simonw/click-app-template-repository/languages", "stargazers_url": "https://api.github.com/repos/simonw/click-app-template-repository/stargazers", "contributors_url": "https://api.github.com/repos/simonw/click-app-template-repository/contributors", "subscribers_url": "https://api.github.com/repos/simonw/click-app-template-repository/subscribers", "subscription_url": "https://api.github.com/repos/simonw/click-app-template-repository/subscription", "commits_url": "https://api.github.com/repos/simonw/click-app-template-repository/commits{/sha}", "git_commits_url": "https://api.github.com/repos/simonw/click-app-template-repository/git/commits{/sha}", "comments_url": "https://api.github.com/repos/simonw/click-app-template-repository/comments{/number}", "issue_comment_url": "https://api.github.com/repos/simonw/click-app-template-repository/issues/comments{/number}", "contents_url": "https://api.github.com/repos/simonw/click-app-template-repository/contents/{+path}", "compare_url": "https://api.github.com/repos/simonw/click-app-template-repository/compare/{base}...{head}", "merges_url": "https://api.github.com/repos/simonw/click-app-template-repository/merges", "archive_url": "https://api.github.com/repos/simonw/click-app-template-repository/{archive_format}{/ref}", "downloads_url": "https://api.github.com/repos/simonw/click-app-template-repository/downloads", "issues_url": "https://api.github.com/repos/simonw/click-app-template-repository/issues{/number}", "pulls_url": "https://api.github.com/repos/simonw/click-app-template-repository/pulls{/number}", "milestones_url": "https://api.github.com/repos/simonw/click-app-template-repository/milestones{/number}", "notifications_url": "https://api.github.com/repos/simonw/click-app-template-repository/notifications{?since,all,participating}", "labels_url": "https://api.github.com/repos/simonw/click-app-template-repository/labels{/name}", "releases_url": "https://api.github.com/repos/simonw/click-app-template-repository/releases{/id}", "deployments_url": "https://api.github.com/repos/simonw/click-app-template-repository/deployments", "created_at": "2021-08-30T01:03:34Z", "updated_at": "2026-04-04T18:38:55Z", "pushed_at": "2024-05-12T17:13:00Z", "git_url": "git://github.com/simonw/click-app-template-repository.git", "ssh_url": "git@github.com:simonw/click-app-template-repository.git", "clone_url": "https://github.com/simonw/click-app-template-repository.git", "svn_url": "https://github.com/simonw/click-app-template-repository", "homepage": "", "size": 19, "stargazers_count": 38, "watchers_count": 38, "language": null, "has_issues": true, "has_projects": true, "has_downloads": true, "has_wiki": true, "has_pages": false, "has_discussions": false, "forks_count": 2, "mirror_url": null, "archived": false, "disabled": false, "open_issues_count": 1, "license": null, "allow_forking": true, "is_template": true, "web_commit_signoff_required": false, "has_pull_requests": true, "pull_request_creation_policy": "all", "topics": [], "visibility": "public", "forks": 2, "open_issues": 1, "watchers": 38, "default_branch": "main", "permissions": {"admin": false, "maintain": false, "push": false, "triage": false, "pull": false}, "temp_clone_token": ""}

readme:
  # git-history

  [![PyPI](https://img.shields.io/pypi/v/git-history.svg)](https://pypi.org/project/git-history/)
  [![Changelog](https://img.shields.io/github/v/release/simonw/git-history?include_prereleases&label=changelog)](https://github.com/simonw/git-history/releases)
  [![Tests](https://github.com/simonw/git-history/workflows/Test/badge.svg)](https://github.com/simonw/git-history/actions?query=workflow%3ATest)
  [![License](https://img.shields.io/badge/license-Apache%202.0-blue.svg)](https://github.com/simonw/git-history/blob/master/LICENSE)

  Tools for analyzing Git history using SQLite

  For background on this project see [git-history: a tool for analyzing scraped data collected using Git and SQLite](https://simonwillison.net/2021/Dec/7/git-history/).

  [Measuring traffic during the Half Moon Bay Pumpkin Festival](https://simonwillison.net/2022/Oct/19/measuring-traffic/) describes a project using this tool in detail.

  ## Installation

  Install this tool using `pip`:

      $ pip install git-history

  ## Demos

  [git-history-demos.datasette.io](http://git-history-demos.datasette.io/) hosts three example databases created using this tool:

  - [pge-outages](https://git-history-demos.datasette.io/pge-outages) shows a history of PG&E (the electricity supplier) [outages](https://pgealerts.alerts.pge.com/outagecenter/), using data collected in [simonw/pge-outages](https://github.com/simonw/pge-outages) converted using [pge-outages.sh](https://github.com/simonw/git-history/blob/main/demos/pge-outages.sh)
  - [ca-fires](https://git-history-demos.datasette.io/ca-fires) shows a history of fires in California reported on [fire.ca.gov/incidents](https://www.fire.ca.gov/incidents/), from data in [simonw/ca-fires-history](https://github.com/simonw/ca-fires-history) converted using [ca-fires.sh](https://github.com/simonw/git-history/blob/main/demos/ca-fires.sh)

  The demos are deployed using [Datasette](https://datasette.io/) on [Google Cloud Run](https://cloud.google.com/run/) by [this GitHub Actions workflow](https://github.com/simonw/git-history/blob/main/.github/workflows/deploy-demos.yml).

  ## Usage

  This tool can be run against a Git repository that holds a file that contains JSON, CSV/TSV or some other format and which has multiple versions tracked in the Git history. Read [Git scraping: track changes over time by scraping to a Git repository](https://simonwillison.net/2020/Oct/9/git-scraping/) to understand how you might create such a repository.

  The `file` command analyzes the history of an individual file within the repository, and generates a SQLite database table that represents the different versions of that file over time.

  The file is assumed to contain multiple objects - for example, the results of scraping an electricity outage map or a CSV file full of records.

  Assuming you have a file called `incidents.json` that is a JSON array of objects, with multiple versions of that file recorded in a repository. Each version of that file might look something like this:

  ```json
  [
      {
          "IncidentID": "abc123",
          "Location": "Corner of 4th and Vermont",
          "Type": "fire"
      },
      {
          "IncidentID": "cde448",
          "Location": "555 West Example Drive",
          "Type": "medical"
      }
  ]
  ```

  Change directory into the GitHub repository in question and run the following:

      git-history file incidents.db incidents.json

  This will create a new SQLite database in the `incidents.db` file with three tables:

  - `commits` containing a row for every commit, with a `hash` column, the `commit_at` date and a foreign key to a `namespace`.
  - `item` containing a row for every item in every version of the `filename.json` file - with an extra `_commit` column that is a foreign key back to the `commit` table.
  - `namespaces` containing a single row. This allows you to build multiple tables for different files, using the `--namespace` option described below.

  The database schema for this example will look like this:

  <!-- [[[cog
  import cog, json
  from git_history import cli
  from click.testing import CliRunner
  from tests.test_git_history import make_repo
  import sqlite_utils
  import tempfile, pathlib
  tmpdir = pathlib.Path(tempfile.mkdtemp())
  db_path = str(tmpdir / "data.db")
  make_repo(tmpdir)
  runner = CliRunner()
  result = runner.invoke(cli.cli, [
      "file", db_path, str(tmpdir / "repo" / "incidents.json"), "--repo", str(tmpdir / "repo")
  ])
  cog.out("```sql\n")
  cog.out(sqlite_utils.Database(db_path).schema)
  cog.out("\n```")
  ]]] -->
  ```sql
  CREATE TABLE [namespaces] (
     [id] INTEGER PRIMARY KEY,
     [name] TEXT
  );
  CREATE UNIQUE INDEX [idx_namespaces_name]
      ON [namespaces] ([name]);
  CREATE TABLE [commits] (
     [id] INTEGER PRIMARY KEY,
     [namespace] INTEGER REFERENCES [namespaces]([id]),
     [hash] TEXT,
     [commit_at] TEXT
  );
  CREATE UNIQUE INDEX [idx_commits_namespace_hash]
      ON [commits] ([namespace], [hash]);
  CREATE TABLE [item] (
     [IncidentID] TEXT,
     [Location] TEXT,
     [Type] TEXT,
     [_commit] INTEGER REFERENCES [commits]([id])
  );
  ```
  <!-- [[[end]]] -->

  If you have 10 historic versions of the `incidents.json` file and each one contains 30 incidents, you will end up with 10 * 30 = 300 rows in your `item` table.

  ### Track the history of individual items using IDs

  If your objects have a unique identifier - or multiple columns that together form a unique identifier - you can use the `--id` option to de-duplicate and track changes to each of those items over time.

  This provides a much more interesting way to apply this tool.

  If there is a unique identifier column called `IncidentID` you could run the following:

      git-history file incidents.db incidents.json --id IncidentID

  The database schema used here is very different from the one used without the `--id` option.

  If you have already imported history, the command will skip any commits that it has seen already and just process new ones. This means that even though an initial import could be slow subsequent imports should run a lot faster.

  This command will create six tables - `commits`, `item`, `item_version`, `columns`, `item_changed` and `namespaces`.

  Here's the full schema:

  <!-- [[[cog
  db_path2 = str(tmpdir / "data2.db")
  result = runner.invoke(cli.cli, [
      "file", db_path2, str(tmpdir / "repo" / "incidents.json"),
      "--repo", str(tmpdir / "repo"),
      "--id", "IncidentID"
  ])
  cog.out("```sql\n")
  cog.out(sqlite_utils.Database(db_path2).schema)
  cog.out("\n```")
  ]]] -->
  ```sql
  CREATE TABLE [namespaces] (
     [id] INTEGER PRIMARY KEY,
     [name] TEXT
  );
  CREATE UNIQUE INDEX [idx_namespaces_name]
      ON [namespaces] ([name]);
  CREATE TABLE [commits] (
     [id] INTEGER PRIMARY KEY,
     [namespace] INTEGER REFERENCES [namespaces]([id]),
     [hash] TEXT,
     [commit_at] TEXT
  );
  CREATE UNIQUE INDEX [idx_commits_namespace_hash]
      ON [commits] ([namespace], [hash]);
  CREATE TABLE [item] (
     [_id] INTEGER PRIMARY KEY,
     [_item_id] TEXT
  , [IncidentID] TEXT, [Location] TEXT, [Type] TEXT, [_commit] INTEGER);
  CREATE UNIQUE INDEX [idx_item__item_id]
      ON [item] ([_item_id]);
  CREATE TABLE [item_version] (
     [_id] INTEGER PRIMARY KEY,
     [_item] INTEGER REFERENCES [item]([_id]),
     [_version] INTEGER,
     [_commit] INTEGER REFERENCES [commits]([id]),
     [IncidentID] TEXT,
     [Location] TEXT,
     [Type] TEXT,
     [_item_full_hash] TEXT
  );
  CREATE TABLE [columns] (
     [id] INTEGER PRIMARY KEY,
     [namespace] INTEGER REFERENCES [namespaces]([id]),
     [name] TEXT
  );
  CREATE UNIQUE INDEX [idx_columns_namespace_name]
      ON [columns] ([namespace], [name]);
  CREATE TABLE [item_changed] (
     [item_version] INTEGER REFERENCES [item_version]([_id]),
     [column] INTEGER REFERENCES [columns]([id]),
     PRIMARY KEY ([item_version], [column])
  );
  CREATE VIEW item_version_detail AS select
    commits.commit_at as _commit_at,
    commits.hash as _commit_hash,
    item_version.*,
    (
      select json_group_array(name) from columns
      where id in (
        select column from item_changed
        where item_version = item_version._id
      )
  ) as _changed_columns
  from item_version
    join commits on commits.id = item_version._commit;
  CREATE INDEX [idx_item_version__item]
      ON [item_version] ([_item]);
  ```
  <!-- [[[end]]] -->

  #### item table

  The `item` table will contain the most recent version of each row, de-duplicated by ID, plus the following additional columns:

  - `_id` - a numeric integer primary key, used as a foreign key from the `item_version` table.
  - `_item_id` - a hash of the values of the columns specified using the `--id` option to the command. This is used for de-duplication when processing new versions.
  - `_commit` - a foreign key to the `commit` table, representing the most recent commit to modify this item.

  #### item_version table

  The `item_version` table will contain a row for each captured differing version of that item, plus the following columns:

  - `_id` - a numeric ID for the item version record.
  - `_item` - a foreign key to the `item` table.
  - `_version` - the numeric version number, starting at 1 and incrementing for each captured version.
  - `_commit` - a foreign key to the `commit` table.
  - `_item_full_hash` - a hash of this version of the item. This is used internally by the tool to identify items that have changed between commits.

  The other columns in this table represent columns in the original data that have changed since the previous version. If the value has not changed, it will be represented by a `null`.

  If a value was previously set but has been changed back to `null` it will still be represented as `null` in the `item_version` row. You can identify these using the `item_changed` many-to-many table described below.

  You can use the `--full-versions` option to store full copies of the item at each version, rather than just storing the columns that have changed.

  #### item_version_detail view

  This SQL view joins `item_version` against `commits` to add three further columns: `_commit_at` with the date of the commit, and `_commit_hash` with the Git commit hash.

  #### item_changed

  This many-to-many table indicates exactly which columns were changed in an `item_version`.

  - `item_version` is a foreign key to a row in the `item_version` table.
  - `column` is a foreign key to a row in the `columns` table.

  This table with have the largest number of rows, which is why it stores just two integers in order to save space.

  #### columns

  The `columns` table stores column names. It is referenced by `item_changed`.

  - `id` - an integer ID.
  - `name` - the name of the column.
  - `namespace` - a foreign key to `namespaces`, for if multiple file histories are sharing the same database.

  #### Reserved column names

  <!-- [[[cog
  from git_history.utils import RESERVED
  cog.out("Note that ")
  cog.out(", ".join("`{}`".format(r) for r in RESERVED))
  cog.out(" are considered reserved column names for the purposes of this tool.")
  ]]] -->
  Note that `_id`, `_item_full_hash`, `_item`, `_item_id`, `_version`, `_commit`, `_item_id`, `_commit_at`, `_commit_hash`, `_changed_columns`, `rowid` are considered reserved column names for the purposes of this tool.
  <!-- [[[end]]] -->

  If your data contains any of these they will be renamed to add a trailing underscore, for example `_id_`, `_item_`, `_version_`, to avoid clashing with the reserved columns.

  If you have a column with a name such as `_commit_` it will be renamed too, adding an additional trailing underscore, so `_commit_` becomes `_commit__` and `_commit__` becomes `_commit___`.

  ### Additional options

  - `--repo DIRECTORY` - the path to the Git repository, if it is not the current working directory.
  - `--branch TEXT` - the Git branch to analyze - defaults to `main`.
  - `--id TEXT` - as described above: pass one or more columns that uniquely identify a record, so that changes to that record can be calculated over time.
  - `--full-versions` - instead of recording just the columns that have changed in the `item_version` table record a full copy of each version of theh item.
  - `--ignore TEXT` - one or more columns to ignore - they will not be included in the resulting database.
  - `--csv` - treat the data is CSV or TSV rather than JSON, and attempt to guess the correct dialect
  - `--dialect` - use a spcific CSV dialect. Options are `excel`, `excel-tab` and `unix` - see [the Python CSV documentation](https://docs.python.org/3/library/csv.html#csv.excel) for details.
  - `--skip TEXT` - one or more full Git commit hashes that should be skipped. You can use this if some of the data in your revision history is corrupted in a way that prevents this tool from working.
  - `--start-at TEXT` - skip commits prior to the specified commit hash.
  - `--start-after TEXT` - skip commits up to and including the specified commit hash, then start processing from the following commit.
  - `--convert TEXT` - custom Python code for a conversion, described below.
  - `--import TEXT` - additional Python modules to import for `--convert`.
  - `--ignore-duplicate-ids` - if a single version of a file has the same ID in it more than once, the tool will exit with an error. Use this option to ignore this and instead pick just the first of the two duplicates.
  - `--namespace TEXT` - use this if you wish to include the history of multiple different files in the same database. The default is `item` but you can set it to something else, which will produce tables with names like `yournamespace` and `yournamespace_version`.
  - `--wal` - Enable WAL mode on the created database file. Use this if you plan to run queries against the database while `git-history` is creating it.
  - `--silent` - don't show the progress bar.

  ### CSV and TSV data

  If the data in your repository is a CSV or TSV file you can process it by adding the `--csv` option. This will attempt to detect which delimiter is used by the file, so the same option works for both comma- and tab-separated values.

      git-history file trees.db trees.csv --id TreeID

  You can also specify the CSV dialect using the `--dialect` option.

  ### Custom conversions using --convert

  If your data is not already either CSV/TSV or a flat JSON array, you can reshape it using the `--convert` option.

  The format needed by this tool is an array of dictionaries, as demonstrated by the `incidents.json` example above.

  If your data does not fit this shape, you can provide a snippet of Python code to converts the on-disk content of each stored file into a Python list of dictionaries.

  For example, if your stored files each look like this:

  ```json
  {
      "incidents": [
          {
              "id": "552",
              "name": "Hawthorne Fire",
              "engines": 3
          },
          {
              "id": "556",
              "name": "Merlin Fire",
              "engines": 1
          }
      ]
  }
  ```
  You could use the following Python snippet to convert them to the required format:

  ```python
  json.loads(content)["incidents"]
  ```
  (The `json` module is exposed to your custom function by default.)

  You would then run the tool like this:

      git-history file database.db incidents.json \
        --id id \
        --convert 'json.loads(content)["incidents"]'

  The `content` variable is always a `bytes` object representing the content of the file at a specific moment in the repository's history.

  You can import additional modules using `--import`. This example shows how you could read a CSV file that uses `;` as the delimiter:

      git-history file trees.db ../sf-tree-history/Street_Tree_List.csv \
        --repo ../sf-tree-history \
        --import csv \
        --import io \
        --convert '
          fp = io.StringIO(content.decode("utf-8"))
          return list(csv.DictReader(fp, delimiter=";"))
          ' \
        --id TreeID

  You can import nested modules such as [ElementTree](https://docs.python.org/3/library/xml.etree.elementtree.html)  using `--import xml.etree.ElementTree`, then refer to them in your function body as `xml.etree.ElementTree`. For example, if your tracked data was in an `items.xml` file that looked like this:

  ```xml
  <items>
    <item id="1" name="One" />
    <item id="2" name="Two" />
    <item id="3" name="Three" />
  </item>
  ```
  You could load it using the following `--convert` script:
  ```
  git-history file items.xml --convert '
  tree = xml.etree.ElementTree.fromstring(content)
  return [el.attrib for el in tree.iter("item")]
  ' --import xml.etree.ElementTree --id id
  ```

  If your Python code spans more than one line it needs to include a `return` statement.

  You can also use Python generators in your `--convert` code, for example:

      git-history file stats.db package-stats/stats.json \
          --repo package-stats \
          --convert '
          data = json.loads(content)
          for key, counts in data.items():
              for date, count in counts.items():
                  yield {
                      "package": key,
                      "date": date,
                      "count": count
                  }
          ' --id package --id date

  This conversion function expects data that looks like this:

  ```json
  {
      "airtable-export": {
          "2021-05-18": 66,
          "2021-05-19": 60,
          "2021-05-20": 87
      }
  }
  ```

  ## Development

  To contribute to this tool, first checkout the code. Then create a new virtual environment:

      cd git-history
      python -m venv venv
      source venv/bin/activate

  Or if you are using `pipenv`:

      pipenv shell

  Now install the dependencies and test dependencies:

      pip install -e '.[test]'

  To run the tests:

      pytest

  To update the schema examples in this README file:

      cog -r README.md

readme_html:
  <div id="readme" class="md" data-path="README.md"><article class="markdown-body entry-content container-lg" itemprop="text"><div class="markdown-heading" dir="auto"><h1 class="heading-element" dir="auto">git-history</h1><a id="user-content-git-history" class="anchor" aria-label="Permalink: git-history" href="#git-history"><svg class="octicon octicon-link" viewbox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
  <p dir="auto"><a href="https://pypi.org/project/git-history/" rel="nofollow"><img src="https://img.shields.io/pypi/v/git-history.svg" alt="PyPI" style="max-width: 100%;" data-camo-src="https://camo.githubusercontent.com/bf58c9407718bd1abca0280d3d738ba1ee64213204fdc4ce52b303dbd6c51996/68747470733a2f2f696d672e736869656c64732e696f2f707970692f762f6769742d686973746f72792e737667"></a>
  <a href="https://github.com/simonw/git-history/releases"><img src="https://img.shields.io/github/v/release/simonw/git-history?include_prereleases&label=changelog" alt="Changelog" style="max-width: 100%;" data-camo-src="https://camo.githubusercontent.com/3d604d42999e194f35b32ea2917eeb43ca0ff5d5a953dd15bb7c8b5c6e4b61e5/68747470733a2f2f696d672e736869656c64732e696f2f6769746875622f762f72656c656173652f73696d6f6e772f6769742d686973746f72793f696e636c7564655f70726572656c6561736573266c6162656c3d6368616e67656c6f67"></a>
  <a href="https://github.com/simonw/git-history/actions?query=workflow%3ATest"><img src="https://github.com/simonw/git-history/workflows/Test/badge.svg" alt="Tests" style="max-width: 100%;"></a>
  <a href="https://github.com/simonw/git-history/blob/master/LICENSE"><img src="https://img.shields.io/badge/license-Apache%202.0-blue.svg" alt="License" style="max-width: 100%;" data-camo-src="https://camo.githubusercontent.com/c355f200ea90fddaa407b6eaab303663a669248ea3ca7b1fcf77dbe04ff5f48c/68747470733a2f2f696d672e736869656c64732e696f2f62616467652f6c6963656e73652d417061636865253230322e302d626c75652e737667"></a></p>
  <p dir="auto">Tools for analyzing Git history using SQLite</p>
  <p dir="auto">For background on this project see <a href="https://simonwillison.net/2021/Dec/7/git-history/" rel="nofollow">git-history: a tool for analyzing scraped data collected using Git and SQLite</a>.</p>
  <p dir="auto"><a href="https://simonwillison.net/2022/Oct/19/measuring-traffic/" rel="nofollow">Measuring traffic during the Half Moon Bay Pumpkin Festival</a> describes a project using this tool in detail.</p>
  <div class="markdown-heading" dir="auto"><h2 class="heading-element" dir="auto">Installation</h2><a id="user-content-installation" class="anchor" aria-label="Permalink: Installation" href="#installation"><svg class="octicon octicon-link" viewbox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
  <p dir="auto">Install this tool using <code>pip</code>:</p>
  <div class="snippet-clipboard-content notranslate position-relative overflow-auto" data-snippet-clipboard-copy-content="$ pip install git-history"><pre class="notranslate"><code>$ pip install git-history
  </code></pre></div>
  <div class="markdown-heading" dir="auto"><h2 class="heading-element" dir="auto">Demos</h2><a id="user-content-demos" class="anchor" aria-label="Permalink: Demos" href="#demos"><svg class="octicon octicon-link" viewbox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
  <p dir="auto"><a href="http://git-history-demos.datasette.io/" rel="nofollow">git-history-demos.datasette.io</a> hosts three example databases created using this tool:</p>
  <ul dir="auto">
  <li><a href="https://git-history-demos.datasette.io/pge-outages" rel="nofollow">pge-outages</a> shows a history of PG&E (the electricity supplier) <a href="https://pgealerts.alerts.pge.com/outagecenter/" rel="nofollow">outages</a>, using data collected in <a href="https://github.com/simonw/pge-outages">simonw/pge-outages</a> converted using <a href="https://github.com/simonw/git-history/blob/main/demos/pge-outages.sh">pge-outages.sh</a></li>
  <li><a href="https://git-history-demos.datasette.io/ca-fires" rel="nofollow">ca-fires</a> shows a history of fires in California reported on <a href="https://www.fire.ca.gov/incidents/" rel="nofollow">fire.ca.gov/incidents</a>, from data in <a href="https://github.com/simonw/ca-fires-history">simonw/ca-fires-history</a> converted using <a href="https://github.com/simonw/git-history/blob/main/demos/ca-fires.sh">ca-fires.sh</a></li>
  </ul>
  <p dir="auto">The demos are deployed using <a href="https://datasette.io/" rel="nofollow">Datasette</a> on <a href="https://cloud.google.com/run/" rel="nofollow">Google Cloud Run</a> by <a href="https://github.com/simonw/git-history/blob/main/.github/workflows/deploy-demos.yml">this GitHub Actions workflow</a>.</p>
  <div class="markdown-heading" dir="auto"><h2 class="heading-element" dir="auto">Usage</h2><a id="user-content-usage" class="anchor" aria-label="Permalink: Usage" href="#usage"><svg class="octicon octicon-link" viewbox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
  <p dir="auto">This tool can be run against a Git repository that holds a file that contains JSON, CSV/TSV or some other format and which has multiple versions tracked in the Git history. Read <a href="https://simonwillison.net/2020/Oct/9/git-scraping/" rel="nofollow">Git scraping: track changes over time by scraping to a Git repository</a> to understand how you might create such a repository.</p>
  <p dir="auto">The <code>file</code> command analyzes the history of an individual file within the repository, and generates a SQLite database table that represents the different versions of that file over time.</p>
  <p dir="auto">The file is assumed to contain multiple objects - for example, the results of scraping an electricity outage map or a CSV file full of records.</p>
  <p dir="auto">Assuming you have a file called <code>incidents.json</code> that is a JSON array of objects, with multiple versions of that file recorded in a repository. Each version of that file might look something like this:</p>
  <div class="highlight highlight-source-json notranslate position-relative overflow-auto" dir="auto" data-snippet-clipboard-copy-content="[
      {
          "IncidentID": "abc123",
          "Location": "Corner of 4th and Vermont",
          "Type": "fire"
      },
      {
          "IncidentID": "cde448",
          "Location": "555 West Example Drive",
          "Type": "medical"
      }
  ]"><pre>[
      {
          <span class="pl-ent">"IncidentID"</span>: <span class="pl-s"><span class="pl-pds">"</span>abc123<span class="pl-pds">"</span></span>,
          <span class="pl-ent">"Location"</span>: <span class="pl-s"><span class="pl-pds">"</span>Corner of 4th and Vermont<span class="pl-pds">"</span></span>,
          <span class="pl-ent">"Type"</span>: <span class="pl-s"><span class="pl-pds">"</span>fire<span class="pl-pds">"</span></span>
      },
      {
          <span class="pl-ent">"IncidentID"</span>: <span class="pl-s"><span class="pl-pds">"</span>cde448<span class="pl-pds">"</span></span>,
          <span class="pl-ent">"Location"</span>: <span class="pl-s"><span class="pl-pds">"</span>555 West Example Drive<span class="pl-pds">"</span></span>,
          <span class="pl-ent">"Type"</span>: <span class="pl-s"><span class="pl-pds">"</span>medical<span class="pl-pds">"</span></span>
      }
  ]</pre></div>
  <p dir="auto">Change directory into the GitHub repository in question and run the following:</p>
  <div class="snippet-clipboard-content notranslate position-relative overflow-auto" data-snippet-clipboard-copy-content="git-history file incidents.db incidents.json"><pre class="notranslate"><code>git-history file incidents.db incidents.json
  </code></pre></div>
  <p dir="auto">This will create a new SQLite database in the <code>incidents.db</code> file with three tables:</p>
  <ul dir="auto">
  <li><code>commits</code> containing a row for every commit, with a <code>hash</code> column, the <code>commit_at</code> date and a foreign key to a <code>namespace</code>.</li>
  <li><code>item</code> containing a row for every item in every version of the <code>filename.json</code> file - with an extra <code>_commit</code> column that is a foreign key back to the <code>commit</code> table.</li>
  <li><code>namespaces</code> containing a single row. This allows you to build multiple tables for different files, using the <code>--namespace</code> option described below.</li>
  </ul>
  <p dir="auto">The database schema for this example will look like this:</p>

  <div class="highlight highlight-source-sql notranslate position-relative overflow-auto" dir="auto" data-snippet-clipboard-copy-content="CREATE TABLE [namespaces] (
     [id] INTEGER PRIMARY KEY,
     [name] TEXT
  );
  CREATE UNIQUE INDEX [idx_namespaces_name]
      ON [namespaces] ([name]);
  CREATE TABLE [commits] (
     [id] INTEGER PRIMARY KEY,
     [namespace] INTEGER REFERENCES [namespaces]([id]),
     [hash] TEXT,
     [commit_at] TEXT
  );
  CREATE UNIQUE INDEX [idx_commits_namespace_hash]
      ON [commits] ([namespace], [hash]);
  CREATE TABLE [item] (
     [IncidentID] TEXT,
     [Location] TEXT,
     [Type] TEXT,
     [_commit] INTEGER REFERENCES [commits]([id])
  );"><pre>CREATE TABLE [namespaces] (
     [id] <span class="pl-k">INTEGER</span> <span class="pl-k">PRIMARY KEY</span>,
     [name] <span class="pl-k">TEXT</span>
  );
  CREATE UNIQUE INDEX [idx_namespaces_name]
      <span class="pl-k">ON</span> [namespaces] ([name]);
  CREATE TABLE [commits] (
     [id] <span class="pl-k">INTEGER</span> <span class="pl-k">PRIMARY KEY</span>,
     [namespace] <span class="pl-k">INTEGER</span> <span class="pl-k">REFERENCES</span> [namespaces]([id]),
     [hash] <span class="pl-k">TEXT</span>,
     [commit_at] <span class="pl-k">TEXT</span>
  );
  CREATE UNIQUE INDEX [idx_commits_namespace_hash]
      <span class="pl-k">ON</span> [commits] ([namespace], [hash]);
  CREATE TABLE [item] (
     [IncidentID] <span class="pl-k">TEXT</span>,
     [Location] <span class="pl-k">TEXT</span>,
     [Type] <span class="pl-k">TEXT</span>,
     [_commit] <span class="pl-k">INTEGER</span> <span class="pl-k">REFERENCES</span> [commits]([id])
  );</pre></div>

  <p dir="auto">If you have 10 historic versions of the <code>incidents.json</code> file and each one contains 30 incidents, you will end up with 10 * 30 = 300 rows in your <code>item</code> table.</p>
  <div class="markdown-heading" dir="auto"><h3 class="heading-element" dir="auto">Track the history of individual items using IDs</h3><a id="user-content-track-the-history-of-individual-items-using-ids" class="anchor" aria-label="Permalink: Track the history of individual items using IDs" href="#track-the-history-of-individual-items-using-ids"><svg class="octicon octicon-link" viewbox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
  <p dir="auto">If your objects have a unique identifier - or multiple columns that together form a unique identifier - you can use the <code>--id</code> option to de-duplicate and track changes to each of those items over time.</p>
  <p dir="auto">This provides a much more interesting way to apply this tool.</p>
  <p dir="auto">If there is a unique identifier column called <code>IncidentID</code> you could run the following:</p>
  <div class="snippet-clipboard-content notranslate position-relative overflow-auto" data-snippet-clipboard-copy-content="git-history file incidents.db incidents.json --id IncidentID"><pre class="notranslate"><code>git-history file incidents.db incidents.json --id IncidentID
  </code></pre></div>
  <p dir="auto">The database schema used here is very different from the one used without the <code>--id</code> option.</p>
  <p dir="auto">If you have already imported history, the command will skip any commits that it has seen already and just process new ones. This means that even though an initial import could be slow subsequent imports should run a lot faster.</p>
  <p dir="auto">This command will create six tables - <code>commits</code>, <code>item</code>, <code>item_version</code>, <code>columns</code>, <code>item_changed</code> and <code>namespaces</code>.</p>
  <p dir="auto">Here's the full schema:</p>

  <div class="highlight highlight-source-sql notranslate position-relative overflow-auto" dir="auto" data-snippet-clipboard-copy-content="CREATE TABLE [namespaces] (
     [id] INTEGER PRIMARY KEY,
     [name] TEXT
  );
  CREATE UNIQUE INDEX [idx_namespaces_name]
      ON [namespaces] ([name]);
  CREATE TABLE [commits] (
     [id] INTEGER PRIMARY KEY,
     [namespace] INTEGER REFERENCES [namespaces]([id]),
     [hash] TEXT,
     [commit_at] TEXT
  );
  CREATE UNIQUE INDEX [idx_commits_namespace_hash]
      ON [commits] ([namespace], [hash]);
  CREATE TABLE [item] (
     [_id] INTEGER PRIMARY KEY,
     [_item_id] TEXT
  , [IncidentID] TEXT, [Location] TEXT, [Type] TEXT, [_commit] INTEGER);
  CREATE UNIQUE INDEX [idx_item__item_id]
      ON [item] ([_item_id]);
  CREATE TABLE [item_version] (
     [_id] INTEGER PRIMARY KEY,
     [_item] INTEGER REFERENCES [item]([_id]),
     [_version] INTEGER,
     [_commit] INTEGER REFERENCES [commits]([id]),
     [IncidentID] TEXT,
     [Location] TEXT,
     [Type] TEXT,
     [_item_full_hash] TEXT
  );
  CREATE TABLE [columns] (
     [id] INTEGER PRIMARY KEY,
     [namespace] INTEGER REFERENCES [namespaces]([id]),
     [name] TEXT
  );
  CREATE UNIQUE INDEX [idx_columns_namespace_name]
      ON [columns] ([namespace], [name]);
  CREATE TABLE [item_changed] (
     [item_version] INTEGER REFERENCES [item_version]([_id]),
     [column] INTEGER REFERENCES [columns]([id]),
     PRIMARY KEY ([item_version], [column])
  );
  CREATE VIEW item_version_detail AS select
    commits.commit_at as _commit_at,
    commits.hash as _commit_hash,
    item_version.*,
    (
      select json_group_array(name) from columns
      where id in (
        select column from item_changed
        where item_version = item_version._id
      )
  ) as _changed_columns
  from item_version
    join commits on commits.id = item_version._commit;
  CREATE INDEX [idx_item_version__item]
      ON [item_version] ([_item]);"><pre>CREATE TABLE [namespaces] (
     [id] <span class="pl-k">INTEGER</span> <span class="pl-k">PRIMARY KEY</span>,
     [name] <span class="pl-k">TEXT</span>
  );
  CREATE UNIQUE INDEX [idx_namespaces_name]
      <span class="pl-k">ON</span> [namespaces] ([name]);
  CREATE TABLE [commits] (
     [id] <span class="pl-k">INTEGER</span> <span class="pl-k">PRIMARY KEY</span>,
     [namespace] <span class="pl-k">INTEGER</span> <span class="pl-k">REFERENCES</span> [namespaces]([id]),
     [hash] <span class="pl-k">TEXT</span>,
     [commit_at] <span class="pl-k">TEXT</span>
  );
  CREATE UNIQUE INDEX [idx_commits_namespace_hash]
      <span class="pl-k">ON</span> [commits] ([namespace], [hash]);
  CREATE TABLE [item] (
     [_id] <span class="pl-k">INTEGER</span> <span class="pl-k">PRIMARY KEY</span>,
     [_item_id] <span class="pl-k">TEXT</span>
  , [IncidentID] <span class="pl-k">TEXT</span>, [Location] <span class="pl-k">TEXT</span>, [Type] <span class="pl-k">TEXT</span>, [_commit] <span class="pl-k">INTEGER</span>);
  CREATE UNIQUE INDEX [idx_item__item_id]
      <span class="pl-k">ON</span> [item] ([_item_id]);
  CREATE TABLE [item_version] (
     [_id] <span class="pl-k">INTEGER</span> <span class="pl-k">PRIMARY KEY</span>,
     [_item] <span class="pl-k">INTEGER</span> <span class="pl-k">REFERENCES</span> [item]([_id]),
     [_version] <span class="pl-k">INTEGER</span>,
     [_commit] <span class="pl-k">INTEGER</span> <span class="pl-k">REFERENCES</span> [commits]([id]),
     [IncidentID] <span class="pl-k">TEXT</span>,
     [Location] <span class="pl-k">TEXT</span>,
     [Type] <span class="pl-k">TEXT</span>,
     [_item_full_hash] <span class="pl-k">TEXT</span>
  );
  CREATE TABLE [columns] (
     [id] <span class="pl-k">INTEGER</span> <span class="pl-k">PRIMARY KEY</span>,
     [namespace] <span class="pl-k">INTEGER</span> <span class="pl-k">REFERENCES</span> [namespaces]([id]),
     [name] <span class="pl-k">TEXT</span>
  );
  CREATE UNIQUE INDEX [idx_columns_namespace_name]
      <span class="pl-k">ON</span> [columns] ([namespace], [name]);
  CREATE TABLE [item_changed] (
     [item_version] <span class="pl-k">INTEGER</span> <span class="pl-k">REFERENCES</span> [item_version]([_id]),
     [column] <span class="pl-k">INTEGER</span> <span class="pl-k">REFERENCES</span> [columns]([id]),
     <span class="pl-k">PRIMARY KEY</span> ([item_version], [column])
  );
  <span class="pl-k">CREATE</span> <span class="pl-k">VIEW</span> <span class="pl-en">item_version_detail</span> <span class="pl-k">AS</span> <span class="pl-k">select</span>
    <span class="pl-c1">commits</span>.<span class="pl-c1">commit_at</span> <span class="pl-k">as</span> _commit_at,
    <span class="pl-c1">commits</span>.<span class="pl-c1">hash</span> <span class="pl-k">as</span> _commit_hash,
    item_version.<span class="pl-k">*</span>,
    (
      <span class="pl-k">select</span> json_group_array(name) <span class="pl-k">from</span> columns
      <span class="pl-k">where</span> id <span class="pl-k">in</span> (
        <span class="pl-k">select</span> column <span class="pl-k">from</span> item_changed
        <span class="pl-k">where</span> item_version <span class="pl-k">=</span> <span class="pl-c1">item_version</span>.<span class="pl-c1">_id</span>
      )
  ) <span class="pl-k">as</span> _changed_columns
  <span class="pl-k">from</span> item_version
    <span class="pl-k">join</span> commits <span class="pl-k">on</span> <span class="pl-c1">commits</span>.<span class="pl-c1">id</span> <span class="pl-k">=</span> <span class="pl-c1">item_version</span>.<span class="pl-c1">_commit</span>;
  CREATE INDEX [idx_item_version__item]
      <span class="pl-k">ON</span> [item_version] ([_item]);</pre></div>

  <div class="markdown-heading" dir="auto"><h4 class="heading-element" dir="auto">item table</h4><a id="user-content-item-table" class="anchor" aria-label="Permalink: item table" href="#item-table"><svg class="octicon octicon-link" viewbox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
  <p dir="auto">The <code>item</code> table will contain the most recent version of each row, de-duplicated by ID, plus the following additional columns:</p>
  <ul dir="auto">
  <li><code>_id</code> - a numeric integer primary key, used as a foreign key from the <code>item_version</code> table.</li>
  <li><code>_item_id</code> - a hash of the values of the columns specified using the <code>--id</code> option to the command. This is used for de-duplication when processing new versions.</li>
  <li><code>_commit</code> - a foreign key to the <code>commit</code> table, representing the most recent commit to modify this item.</li>
  </ul>
  <div class="markdown-heading" dir="auto"><h4 class="heading-element" dir="auto">item_version table</h4><a id="user-content-item_version-table" class="anchor" aria-label="Permalink: item_version table" href="#item_version-table"><svg class="octicon octicon-link" viewbox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
  <p dir="auto">The <code>item_version</code> table will contain a row for each captured differing version of that item, plus the following columns:</p>
  <ul dir="auto">
  <li><code>_id</code> - a numeric ID for the item version record.</li>
  <li><code>_item</code> - a foreign key to the <code>item</code> table.</li>
  <li><code>_version</code> - the numeric version number, starting at 1 and incrementing for each captured version.</li>
  <li><code>_commit</code> - a foreign key to the <code>commit</code> table.</li>
  <li><code>_item_full_hash</code> - a hash of this version of the item. This is used internally by the tool to identify items that have changed between commits.</li>
  </ul>
  <p dir="auto">The other columns in this table represent columns in the original data that have changed since the previous version. If the value has not changed, it will be represented by a <code>null</code>.</p>
  <p dir="auto">If a value was previously set but has been changed back to <code>null</code> it will still be represented as <code>null</code> in the <code>item_version</code> row. You can identify these using the <code>item_changed</code> many-to-many table described below.</p>
  <p dir="auto">You can use the <code>--full-versions</code> option to store full copies of the item at each version, rather than just storing the columns that have changed.</p>
  <div class="markdown-heading" dir="auto"><h4 class="heading-element" dir="auto">item_version_detail view</h4><a id="user-content-item_version_detail-view" class="anchor" aria-label="Permalink: item_version_detail view" href="#item_version_detail-view"><svg class="octicon octicon-link" viewbox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
  <p dir="auto">This SQL view joins <code>item_version</code> against <code>commits</code> to add three further columns: <code>_commit_at</code> with the date of the commit, and <code>_commit_hash</code> with the Git commit hash.</p>
  <div class="markdown-heading" dir="auto"><h4 class="heading-element" dir="auto">item_changed</h4><a id="user-content-item_changed" class="anchor" aria-label="Permalink: item_changed" href="#item_changed"><svg class="octicon octicon-link" viewbox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
  <p dir="auto">This many-to-many table indicates exactly which columns were changed in an <code>item_version</code>.</p>
  <ul dir="auto">
  <li><code>item_version</code> is a foreign key to a row in the <code>item_version</code> table.</li>
  <li><code>column</code> is a foreign key to a row in the <code>columns</code> table.</li>
  </ul>
  <p dir="auto">This table with have the largest number of rows, which is why it stores just two integers in order to save space.</p>
  <div class="markdown-heading" dir="auto"><h4 class="heading-element" dir="auto">columns</h4><a id="user-content-columns" class="anchor" aria-label="Permalink: columns" href="#columns"><svg class="octicon octicon-link" viewbox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
  <p dir="auto">The <code>columns</code> table stores column names. It is referenced by <code>item_changed</code>.</p>
  <ul dir="auto">
  <li><code>id</code> - an integer ID.</li>
  <li><code>name</code> - the name of the column.</li>
  <li><code>namespace</code> - a foreign key to <code>namespaces</code>, for if multiple file histories are sharing the same database.</li>
  </ul>
  <div class="markdown-heading" dir="auto"><h4 class="heading-element" dir="auto">Reserved column names</h4><a id="user-content-reserved-column-names" class="anchor" aria-label="Permalink: Reserved column names" href="#reserved-column-names"><svg class="octicon octicon-link" viewbox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>

  <p dir="auto">Note that <code>_id</code>, <code>_item_full_hash</code>, <code>_item</code>, <code>_item_id</code>, <code>_version</code>, <code>_commit</code>, <code>_item_id</code>, <code>_commit_at</code>, <code>_commit_hash</code>, <code>_changed_columns</code>, <code>rowid</code> are considered reserved column names for the purposes of this tool.</p>

  <p dir="auto">If your data contains any of these they will be renamed to add a trailing underscore, for example <code>_id_</code>, <code>_item_</code>, <code>_version_</code>, to avoid clashing with the reserved columns.</p>
  <p dir="auto">If you have a column with a name such as <code>_commit_</code> it will be renamed too, adding an additional trailing underscore, so <code>_commit_</code> becomes <code>_commit__</code> and <code>_commit__</code> becomes <code>_commit___</code>.</p>
  <div class="markdown-heading" dir="auto"><h3 class="heading-element" dir="auto">Additional options</h3><a id="user-content-additional-options" class="anchor" aria-label="Permalink: Additional options" href="#additional-options"><svg class="octicon octicon-link" viewbox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
  <ul dir="auto">
  <li><code>--repo DIRECTORY</code> - the path to the Git repository, if it is not the current working directory.</li>
  <li><code>--branch TEXT</code> - the Git branch to analyze - defaults to <code>main</code>.</li>
  <li><code>--id TEXT</code> - as described above: pass one or more columns that uniquely identify a record, so that changes to that record can be calculated over time.</li>
  <li><code>--full-versions</code> - instead of recording just the columns that have changed in the <code>item_version</code> table record a full copy of each version of theh item.</li>
  <li><code>--ignore TEXT</code> - one or more columns to ignore - they will not be included in the resulting database.</li>
  <li><code>--csv</code> - treat the data is CSV or TSV rather than JSON, and attempt to guess the correct dialect</li>
  <li><code>--dialect</code> - use a spcific CSV dialect. Options are <code>excel</code>, <code>excel-tab</code> and <code>unix</code> - see <a href="https://docs.python.org/3/library/csv.html#csv.excel" rel="nofollow">the Python CSV documentation</a> for details.</li>
  <li><code>--skip TEXT</code> - one or more full Git commit hashes that should be skipped. You can use this if some of the data in your revision history is corrupted in a way that prevents this tool from working.</li>
  <li><code>--start-at TEXT</code> - skip commits prior to the specified commit hash.</li>
  <li><code>--start-after TEXT</code> - skip commits up to and including the specified commit hash, then start processing from the following commit.</li>
  <li><code>--convert TEXT</code> - custom Python code for a conversion, described below.</li>
  <li><code>--import TEXT</code> - additional Python modules to import for <code>--convert</code>.</li>
  <li><code>--ignore-duplicate-ids</code> - if a single version of a file has the same ID in it more than once, the tool will exit with an error. Use this option to ignore this and instead pick just the first of the two duplicates.</li>
  <li><code>--namespace TEXT</code> - use this if you wish to include the history of multiple different files in the same database. The default is <code>item</code> but you can set it to something else, which will produce tables with names like <code>yournamespace</code> and <code>yournamespace_version</code>.</li>
  <li><code>--wal</code> - Enable WAL mode on the created database file. Use this if you plan to run queries against the database while <code>git-history</code> is creating it.</li>
  <li><code>--silent</code> - don't show the progress bar.</li>
  </ul>
  <div class="markdown-heading" dir="auto"><h3 class="heading-element" dir="auto">CSV and TSV data</h3><a id="user-content-csv-and-tsv-data" class="anchor" aria-label="Permalink: CSV and TSV data" href="#csv-and-tsv-data"><svg class="octicon octicon-link" viewbox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
  <p dir="auto">If the data in your repository is a CSV or TSV file you can process it by adding the <code>--csv</code> option. This will attempt to detect which delimiter is used by the file, so the same option works for both comma- and tab-separated values.</p>
  <div class="snippet-clipboard-content notranslate position-relative overflow-auto" data-snippet-clipboard-copy-content="git-history file trees.db trees.csv --id TreeID"><pre class="notranslate"><code>git-history file trees.db trees.csv --id TreeID
  </code></pre></div>
  <p dir="auto">You can also specify the CSV dialect using the <code>--dialect</code> option.</p>
  <div class="markdown-heading" dir="auto"><h3 class="heading-element" dir="auto">Custom conversions using --convert</h3><a id="user-content-custom-conversions-using---convert" class="anchor" aria-label="Permalink: Custom conversions using --convert" href="#custom-conversions-using---convert"><svg class="octicon octicon-link" viewbox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
  <p dir="auto">If your data is not already either CSV/TSV or a flat JSON array, you can reshape it using the <code>--convert</code> option.</p>
  <p dir="auto">The format needed by this tool is an array of dictionaries, as demonstrated by the <code>incidents.json</code> example above.</p>
  <p dir="auto">If your data does not fit this shape, you can provide a snippet of Python code to converts the on-disk content of each stored file into a Python list of dictionaries.</p>
  <p dir="auto">For example, if your stored files each look like this:</p>
  <div class="highlight highlight-source-json notranslate position-relative overflow-auto" dir="auto" data-snippet-clipboard-copy-content="{
      "incidents": [
          {
              "id": "552",
              "name": "Hawthorne Fire",
              "engines": 3
          },
          {
              "id": "556",
              "name": "Merlin Fire",
              "engines": 1
          }
      ]
  }"><pre>{
      <span class="pl-ent">"incidents"</span>: [
          {
              <span class="pl-ent">"id"</span>: <span class="pl-s"><span class="pl-pds">"</span>552<span class="pl-pds">"</span></span>,
              <span class="pl-ent">"name"</span>: <span class="pl-s"><span class="pl-pds">"</span>Hawthorne Fire<span class="pl-pds">"</span></span>,
              <span class="pl-ent">"engines"</span>: <span class="pl-c1">3</span>
          },
          {
              <span class="pl-ent">"id"</span>: <span class="pl-s"><span class="pl-pds">"</span>556<span class="pl-pds">"</span></span>,
              <span class="pl-ent">"name"</span>: <span class="pl-s"><span class="pl-pds">"</span>Merlin Fire<span class="pl-pds">"</span></span>,
              <span class="pl-ent">"engines"</span>: <span class="pl-c1">1</span>
          }
      ]
  }</pre></div>
  <p dir="auto">You could use the following Python snippet to convert them to the required format:</p>
  <div class="highlight highlight-source-python notranslate position-relative overflow-auto" dir="auto" data-snippet-clipboard-copy-content="json.loads(content)["incidents"]"><pre><span class="pl-s1">json</span>.<span class="pl-c1">loads</span>(<span class="pl-s1">content</span>)[<span class="pl-s">"incidents"</span>]</pre></div>
  <p dir="auto">(The <code>json</code> module is exposed to your custom function by default.)</p>
  <p dir="auto">You would then run the tool like this:</p>
  <div class="snippet-clipboard-content notranslate position-relative overflow-auto" data-snippet-clipboard-copy-content="git-history file database.db incidents.json \
    --id id \
    --convert 'json.loads(content)["incidents"]'"><pre class="notranslate"><code>git-history file database.db incidents.json \
    --id id \
    --convert 'json.loads(content)["incidents"]'
  </code></pre></div>
  <p dir="auto">The <code>content</code> variable is always a <code>bytes</code> object representing the content of the file at a specific moment in the repository's history.</p>
  <p dir="auto">You can import additional modules using <code>--import</code>. This example shows how you could read a CSV file that uses <code>;</code> as the delimiter:</p>
  <div class="snippet-clipboard-content notranslate position-relative overflow-auto" data-snippet-clipboard-copy-content="git-history file trees.db ../sf-tree-history/Street_Tree_List.csv \
    --repo ../sf-tree-history \
    --import csv \
    --import io \
    --convert '
      fp = io.StringIO(content.decode("utf-8"))
      return list(csv.DictReader(fp, delimiter=";"))
      ' \
    --id TreeID"><pre class="notranslate"><code>git-history file trees.db ../sf-tree-history/Street_Tree_List.csv \
    --repo ../sf-tree-history \
    --import csv \
    --import io \
    --convert '
      fp = io.StringIO(content.decode("utf-8"))
      return list(csv.DictReader(fp, delimiter=";"))
      ' \
    --id TreeID
  </code></pre></div>
  <p dir="auto">You can import nested modules such as <a href="https://docs.python.org/3/library/xml.etree.elementtree.html" rel="nofollow">ElementTree</a>  using <code>--import xml.etree.ElementTree</code>, then refer to them in your function body as <code>xml.etree.ElementTree</code>. For example, if your tracked data was in an <code>items.xml</code> file that looked like this:</p>
  <div class="highlight highlight-text-xml notranslate position-relative overflow-auto" dir="auto" data-snippet-clipboard-copy-content="<items>
    <item id="1" name="One" />
    <item id="2" name="Two" />
    <item id="3" name="Three" />
  </item>"><pre><<span class="pl-ent">items</span>>
    <<span class="pl-ent">item</span> <span class="pl-e">id</span>=<span class="pl-s"><span class="pl-pds">"</span>1<span class="pl-pds">"</span></span> <span class="pl-e">name</span>=<span class="pl-s"><span class="pl-pds">"</span>One<span class="pl-pds">"</span></span> />
    <<span class="pl-ent">item</span> <span class="pl-e">id</span>=<span class="pl-s"><span class="pl-pds">"</span>2<span class="pl-pds">"</span></span> <span class="pl-e">name</span>=<span class="pl-s"><span class="pl-pds">"</span>Two<span class="pl-pds">"</span></span> />
    <<span class="pl-ent">item</span> <span class="pl-e">id</span>=<span class="pl-s"><span class="pl-pds">"</span>3<span class="pl-pds">"</span></span> <span class="pl-e">name</span>=<span class="pl-s"><span class="pl-pds">"</span>Three<span class="pl-pds">"</span></span> />
  </<span class="pl-ent">item</span>></pre></div>
  <p dir="auto">You could load it using the following <code>--convert</code> script:</p>
  <div class="snippet-clipboard-content notranslate position-relative overflow-auto" data-snippet-clipboard-copy-content="git-history file items.xml --convert '
  tree = xml.etree.ElementTree.fromstring(content)
  return [el.attrib for el in tree.iter("item")]
  ' --import xml.etree.ElementTree --id id"><pre class="notranslate"><code>git-history file items.xml --convert '
  tree = xml.etree.ElementTree.fromstring(content)
  return [el.attrib for el in tree.iter("item")]
  ' --import xml.etree.ElementTree --id id
  </code></pre></div>
  <p dir="auto">If your Python code spans more than one line it needs to include a <code>return</code> statement.</p>
  <p dir="auto">You can also use Python generators in your <code>--convert</code> code, for example:</p>
  <div class="snippet-clipboard-content notranslate position-relative overflow-auto" data-snippet-clipboard-copy-content="git-history file stats.db package-stats/stats.json \
      --repo package-stats \
      --convert '
      data = json.loads(content)
      for key, counts in data.items():
          for date, count in counts.items():
              yield {
                  "package": key,
                  "date": date,
                  "count": count
              }
      ' --id package --id date"><pre class="notranslate"><code>git-history file stats.db package-stats/stats.json \
      --repo package-stats \
      --convert '
      data = json.loads(content)
      for key, counts in data.items():
          for date, count in counts.items():
              yield {
                  "package": key,
                  "date": date,
                  "count": count
              }
      ' --id package --id date
  </code></pre></div>
  <p dir="auto">This conversion function expects data that looks like this:</p>
  <div class="highlight highlight-source-json notranslate position-relative overflow-auto" dir="auto" data-snippet-clipboard-copy-content="{
      "airtable-export": {
          "2021-05-18": 66,
          "2021-05-19": 60,
          "2021-05-20": 87
      }
  }"><pre>{
      <span class="pl-ent">"airtable-export"</span>: {
          <span class="pl-ent">"2021-05-18"</span>: <span class="pl-c1">66</span>,
          <span class="pl-ent">"2021-05-19"</span>: <span class="pl-c1">60</span>,
          <span class="pl-ent">"2021-05-20"</span>: <span class="pl-c1">87</span>
      }
  }</pre></div>
  <div class="markdown-heading" dir="auto"><h2 class="heading-element" dir="auto">Development</h2><a id="user-content-development" class="anchor" aria-label="Permalink: Development" href="#development"><svg class="octicon octicon-link" viewbox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
  <p dir="auto">To contribute to this tool, first checkout the code. Then create a new virtual environment:</p>
  <div class="snippet-clipboard-content notranslate position-relative overflow-auto" data-snippet-clipboard-copy-content="cd git-history
  python -m venv venv
  source venv/bin/activate"><pre class="notranslate"><code>cd git-history
  python -m venv venv
  source venv/bin/activate
  </code></pre></div>
  <p dir="auto">Or if you are using <code>pipenv</code>:</p>
  <div class="snippet-clipboard-content notranslate position-relative overflow-auto" data-snippet-clipboard-copy-content="pipenv shell"><pre class="notranslate"><code>pipenv shell
  </code></pre></div>
  <p dir="auto">Now install the dependencies and test dependencies:</p>
  <div class="snippet-clipboard-content notranslate position-relative overflow-auto" data-snippet-clipboard-copy-content="pip install -e '.[test]'"><pre class="notranslate"><code>pip install -e '.[test]'
  </code></pre></div>
  <p dir="auto">To run the tests:</p>
  <div class="snippet-clipboard-content notranslate position-relative overflow-auto" data-snippet-clipboard-copy-content="pytest"><pre class="notranslate"><code>pytest
  </code></pre></div>
  <p dir="auto">To update the schema examples in this README file:</p>
  <div class="snippet-clipboard-content notranslate position-relative overflow-auto" data-snippet-clipboard-copy-content="cog -r README.md"><pre class="notranslate"><code>cog -r README.md
  </code></pre></div>
  </article></div>


node_id: R_kgDOGaS1TA
name: datasette-redirect-to-https
full_name: simonw/datasette-redirect-to-https
html_url: https://github.com/simonw/datasette-redirect-to-https
description: Datasette plugin that redirects all non-https requests to https
created_at: 2021-11-20T22:43:33Z
updated_at: 2022-04-24T03:48:01Z
pushed_at: 2022-07-07T17:38:32Z
language: Python
topics: ["asgi", "datasette", "datasette-io", "datasette-plugin"]
visibility: public
default_branch: main
permissions: {"admin": false, "maintain": false, "push": false, "triage": false, "pull": true}
template_repository:
  {"id": 400878073, "node_id": "MDEwOlJlcG9zaXRvcnk0MDA4NzgwNzM=", "name": "datasette-plugin-template-repository", "full_name": "simonw/datasette-plugin-template-repository", "private": false, "owner": {"login": "simonw", "id": 9599, "node_id": "MDQ6VXNlcjk1OTk=", "avatar_url": "https://avatars.githubusercontent.com/u/9599?v=4", "gravatar_id": "", "url": "https://api.github.com/users/simonw", "html_url": "https://github.com/simonw", "followers_url": "https://api.github.com/users/simonw/followers", "following_url": "https://api.github.com/users/simonw/following{/other_user}", "gists_url": "https://api.github.com/users/simonw/gists{/gist_id}", "starred_url": "https://api.github.com/users/simonw/starred{/owner}{/repo}", "subscriptions_url": "https://api.github.com/users/simonw/subscriptions", "organizations_url": "https://api.github.com/users/simonw/orgs", "repos_url": "https://api.github.com/users/simonw/repos", "events_url": "https://api.github.com/users/simonw/events{/privacy}", "received_events_url": "https://api.github.com/users/simonw/received_events", "type": "User", "user_view_type": "public", "site_admin": false}, "html_url": "https://github.com/simonw/datasette-plugin-template-repository", "description": "GitHub template repository for creating new Datasette plugins, using the simonw/datasette-plugin cookiecutter template", "fork": false, "url": "https://api.github.com/repos/simonw/datasette-plugin-template-repository", "forks_url": "https://api.github.com/repos/simonw/datasette-plugin-template-repository/forks", "keys_url": "https://api.github.com/repos/simonw/datasette-plugin-template-repository/keys{/key_id}", "collaborators_url": "https://api.github.com/repos/simonw/datasette-plugin-template-repository/collaborators{/collaborator}", "teams_url": "https://api.github.com/repos/simonw/datasette-plugin-template-repository/teams", "hooks_url": "https://api.github.com/repos/simonw/datasette-plugin-template-repository/hooks", "issue_events_url": "https://api.github.com/repos/simonw/datasette-plugin-template-repository/issues/events{/number}", "events_url": "https://api.github.com/repos/simonw/datasette-plugin-template-repository/events", "assignees_url": "https://api.github.com/repos/simonw/datasette-plugin-template-repository/assignees{/user}", "branches_url": "https://api.github.com/repos/simonw/datasette-plugin-template-repository/branches{/branch}", "tags_url": "https://api.github.com/repos/simonw/datasette-plugin-template-repository/tags", "blobs_url": "https://api.github.com/repos/simonw/datasette-plugin-template-repository/git/blobs{/sha}", "git_tags_url": "https://api.github.com/repos/simonw/datasette-plugin-template-repository/git/tags{/sha}", "git_refs_url": "https://api.github.com/repos/simonw/datasette-plugin-template-repository/git/refs{/sha}", "trees_url": "https://api.github.com/repos/simonw/datasette-plugin-template-repository/git/trees{/sha}", "statuses_url": "https://api.github.com/repos/simonw/datasette-plugin-template-repository/statuses/{sha}", "languages_url": "https://api.github.com/repos/simonw/datasette-plugin-template-repository/languages", "stargazers_url": "https://api.github.com/repos/simonw/datasette-plugin-template-repository/stargazers", "contributors_url": "https://api.github.com/repos/simonw/datasette-plugin-template-repository/contributors", "subscribers_url": "https://api.github.com/repos/simonw/datasette-plugin-template-repository/subscribers", "subscription_url": "https://api.github.com/repos/simonw/datasette-plugin-template-repository/subscription", "commits_url": "https://api.github.com/repos/simonw/datasette-plugin-template-repository/commits{/sha}", "git_commits_url": "https://api.github.com/repos/simonw/datasette-plugin-template-repository/git/commits{/sha}", "comments_url": "https://api.github.com/repos/simonw/datasette-plugin-template-repository/comments{/number}", "issue_comment_url": "https://api.github.com/repos/simonw/datasette-plugin-template-repository/issues/comments{/number}", "contents_url": "https://api.github.com/repos/simonw/datasette-plugin-template-repository/contents/{+path}", "compare_url": "https://api.github.com/repos/simonw/datasette-plugin-template-repository/compare/{base}...{head}", "merges_url": "https://api.github.com/repos/simonw/datasette-plugin-template-repository/merges", "archive_url": "https://api.github.com/repos/simonw/datasette-plugin-template-repository/{archive_format}{/ref}", "downloads_url": "https://api.github.com/repos/simonw/datasette-plugin-template-repository/downloads", "issues_url": "https://api.github.com/repos/simonw/datasette-plugin-template-repository/issues{/number}", "pulls_url": "https://api.github.com/repos/simonw/datasette-plugin-template-repository/pulls{/number}", "milestones_url": "https://api.github.com/repos/simonw/datasette-plugin-template-repository/milestones{/number}", "notifications_url": "https://api.github.com/repos/simonw/datasette-plugin-template-repository/notifications{?since,all,participating}", "labels_url": "https://api.github.com/repos/simonw/datasette-plugin-template-repository/labels{/name}", "releases_url": "https://api.github.com/repos/simonw/datasette-plugin-template-repository/releases{/id}", "deployments_url": "https://api.github.com/repos/simonw/datasette-plugin-template-repository/deployments", "created_at": "2021-08-28T19:50:28Z", "updated_at": "2025-02-18T20:42:29Z", "pushed_at": "2025-02-18T20:42:25Z", "git_url": "git://github.com/simonw/datasette-plugin-template-repository.git", "ssh_url": "git@github.com:simonw/datasette-plugin-template-repository.git", "clone_url": "https://github.com/simonw/datasette-plugin-template-repository.git", "svn_url": "https://github.com/simonw/datasette-plugin-template-repository", "homepage": "", "size": 24, "stargazers_count": 24, "watchers_count": 24, "language": null, "has_issues": true, "has_projects": true, "has_downloads": true, "has_wiki": true, "has_pages": false, "has_discussions": false, "forks_count": 0, "mirror_url": null, "archived": false, "disabled": false, "open_issues_count": 0, "license": null, "allow_forking": true, "is_template": true, "web_commit_signoff_required": false, "topics": [], "visibility": "public", "forks": 0, "open_issues": 0, "watchers": 24, "default_branch": "main", "permissions": {"admin": false, "maintain": false, "push": false, "triage": false, "pull": true}}

readme:
  # datasette-redirect-to-https

  [![PyPI](https://img.shields.io/pypi/v/datasette-redirect-to-https.svg)](https://pypi.org/project/datasette-redirect-to-https/)
  [![Changelog](https://img.shields.io/github/v/release/simonw/datasette-redirect-to-https?include_prereleases&label=changelog)](https://github.com/simonw/datasette-redirect-to-https/releases)
  [![Tests](https://github.com/simonw/datasette-redirect-to-https/workflows/Test/badge.svg)](https://github.com/simonw/datasette-redirect-to-https/actions?query=workflow%3ATest)
  [![License](https://img.shields.io/badge/license-Apache%202.0-blue.svg)](https://github.com/simonw/datasette-redirect-to-https/blob/main/LICENSE)

  Datasette plugin that redirects all non-https requests to https

  ## Installation

  Install this plugin in the same environment as Datasette.

      $ datasette install datasette-redirect-to-https

  ## Usage

  Once installed, incoming GET requests to the `http://` protocol will be 301 redirected to the `https://` equivalent page.

  HTTP verbs other than GET will get a 405 Method Not Allowed HTTP error.

  ## Configuration

  Some hosting providers handle HTTPS for you, passing requests back to your application server over HTTP.

  For this plugin to work correctly, it needs to detect that the original incoming request came in over HTTP.

  Hosting providers like this often set an additional HTTP header such as `x-forwarded-proto: http` identifying the original protocol.

  You can configure `datasette-redirect-to-https` to respect this header using the following plugin configuration in `metadata.json`:

  ```json
  {
    "plugins": {
      "datasette-redirect-to-https": {
        "if_headers": {
          "x-forwarded-proto": "http"
        }
      }
    }
  }
  ```
  The above example will redirect to `https://` if the incoming request has a `x-forwarded-proto: http` request header.

  If multiple `if_headers` are listed, the redirect will occur if any of them match.

  ## Development

  To set up this plugin locally, first checkout the code. Then create a new virtual environment:

      cd datasette-redirect-to-https
      python3 -m venv venv
      source venv/bin/activate

  Now install the dependencies and test dependencies:

      pip install -e '.[test]'

  To run the tests:

      pytest

readme_html:
  <div id="readme" class="md" data-path="README.md"><article class="markdown-body entry-content container-lg" itemprop="text"><div class="markdown-heading" dir="auto"><h1 class="heading-element" dir="auto">datasette-redirect-to-https</h1><a id="user-content-datasette-redirect-to-https" class="anchor" aria-label="Permalink: datasette-redirect-to-https" href="#datasette-redirect-to-https"><svg class="octicon octicon-link" viewbox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
  <p dir="auto"><a href="https://pypi.org/project/datasette-redirect-to-https/" rel="nofollow"><img src="https://img.shields.io/pypi/v/datasette-redirect-to-https.svg" alt="PyPI" style="max-width: 100%;" data-camo-src="https://camo.githubusercontent.com/2b594a6c861d87978741063af4417a25aadc2557ab858d934410e1c37bc3ca34/68747470733a2f2f696d672e736869656c64732e696f2f707970692f762f6461746173657474652d72656469726563742d746f2d68747470732e737667"></a>
  <a href="https://github.com/simonw/datasette-redirect-to-https/releases"><img src="https://img.shields.io/github/v/release/simonw/datasette-redirect-to-https?include_prereleases&label=changelog" alt="Changelog" style="max-width: 100%;" data-camo-src="https://camo.githubusercontent.com/8132ae62207c86a5a71c04cd48dbdb8640968c817a636b333b6331cc6095b7c4/68747470733a2f2f696d672e736869656c64732e696f2f6769746875622f762f72656c656173652f73696d6f6e772f6461746173657474652d72656469726563742d746f2d68747470733f696e636c7564655f70726572656c6561736573266c6162656c3d6368616e67656c6f67"></a>
  <a href="https://github.com/simonw/datasette-redirect-to-https/actions?query=workflow%3ATest"><img src="https://github.com/simonw/datasette-redirect-to-https/workflows/Test/badge.svg" alt="Tests" style="max-width: 100%;"></a>
  <a href="https://github.com/simonw/datasette-redirect-to-https/blob/main/LICENSE"><img src="https://img.shields.io/badge/license-Apache%202.0-blue.svg" alt="License" style="max-width: 100%;" data-camo-src="https://camo.githubusercontent.com/c355f200ea90fddaa407b6eaab303663a669248ea3ca7b1fcf77dbe04ff5f48c/68747470733a2f2f696d672e736869656c64732e696f2f62616467652f6c6963656e73652d417061636865253230322e302d626c75652e737667"></a></p>
  <p dir="auto">Datasette plugin that redirects all non-https requests to https</p>
  <div class="markdown-heading" dir="auto"><h2 class="heading-element" dir="auto">Installation</h2><a id="user-content-installation" class="anchor" aria-label="Permalink: Installation" href="#installation"><svg class="octicon octicon-link" viewbox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
  <p dir="auto">Install this plugin in the same environment as Datasette.</p>
  <div class="snippet-clipboard-content notranslate position-relative overflow-auto" data-snippet-clipboard-copy-content="$ datasette install datasette-redirect-to-https"><pre class="notranslate"><code>$ datasette install datasette-redirect-to-https
  </code></pre></div>
  <div class="markdown-heading" dir="auto"><h2 class="heading-element" dir="auto">Usage</h2><a id="user-content-usage" class="anchor" aria-label="Permalink: Usage" href="#usage"><svg class="octicon octicon-link" viewbox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
  <p dir="auto">Once installed, incoming GET requests to the <code>http://</code> protocol will be 301 redirected to the <code>https://</code> equivalent page.</p>
  <p dir="auto">HTTP verbs other than GET will get a 405 Method Not Allowed HTTP error.</p>
  <div class="markdown-heading" dir="auto"><h2 class="heading-element" dir="auto">Configuration</h2><a id="user-content-configuration" class="anchor" aria-label="Permalink: Configuration" href="#configuration"><svg class="octicon octicon-link" viewbox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
  <p dir="auto">Some hosting providers handle HTTPS for you, passing requests back to your application server over HTTP.</p>
  <p dir="auto">For this plugin to work correctly, it needs to detect that the original incoming request came in over HTTP.</p>
  <p dir="auto">Hosting providers like this often set an additional HTTP header such as <code>x-forwarded-proto: http</code> identifying the original protocol.</p>
  <p dir="auto">You can configure <code>datasette-redirect-to-https</code> to respect this header using the following plugin configuration in <code>metadata.json</code>:</p>
  <div class="highlight highlight-source-json notranslate position-relative overflow-auto" dir="auto" data-snippet-clipboard-copy-content="{
    "plugins": {
      "datasette-redirect-to-https": {
        "if_headers": {
          "x-forwarded-proto": "http"
        }
      }
    }
  }"><pre>{
    <span class="pl-ent">"plugins"</span>: {
      <span class="pl-ent">"datasette-redirect-to-https"</span>: {
        <span class="pl-ent">"if_headers"</span>: {
          <span class="pl-ent">"x-forwarded-proto"</span>: <span class="pl-s"><span class="pl-pds">"</span>http<span class="pl-pds">"</span></span>
        }
      }
    }
  }</pre></div>
  <p dir="auto">The above example will redirect to <code>https://</code> if the incoming request has a <code>x-forwarded-proto: http</code> request header.</p>
  <p dir="auto">If multiple <code>if_headers</code> are listed, the redirect will occur if any of them match.</p>
  <div class="markdown-heading" dir="auto"><h2 class="heading-element" dir="auto">Development</h2><a id="user-content-development" class="anchor" aria-label="Permalink: Development" href="#development"><svg class="octicon octicon-link" viewbox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
  <p dir="auto">To set up this plugin locally, first checkout the code. Then create a new virtual environment:</p>
  <div class="snippet-clipboard-content notranslate position-relative overflow-auto" data-snippet-clipboard-copy-content="cd datasette-redirect-to-https
  python3 -m venv venv
  source venv/bin/activate"><pre class="notranslate"><code>cd datasette-redirect-to-https
  python3 -m venv venv
  source venv/bin/activate
  </code></pre></div>
  <p dir="auto">Now install the dependencies and test dependencies:</p>
  <div class="snippet-clipboard-content notranslate position-relative overflow-auto" data-snippet-clipboard-copy-content="pip install -e '.[test]'"><pre class="notranslate"><code>pip install -e '.[test]'
  </code></pre></div>
  <p dir="auto">To run the tests:</p>
  <div class="snippet-clipboard-content notranslate position-relative overflow-auto" data-snippet-clipboard-copy-content="pytest"><pre class="notranslate"><code>pytest
  </code></pre></div>
  </article></div>


node_id: R_kgDOGeMHbg
name: datasette-hovercards
full_name: simonw/datasette-hovercards
html_url: https://github.com/simonw/datasette-hovercards
description: Add preview hovercards to links in Datasette
created_at: 2021-12-02T17:11:59Z
updated_at: 2022-04-27T15:06:11Z
pushed_at: 2021-12-02T19:57:32Z
language: JavaScript
topics: []
visibility: public
default_branch: main
permissions: {"admin": false, "maintain": false, "push": false, "triage": false, "pull": true}
readme:
  # datasette-hovercards

  [![PyPI](https://img.shields.io/pypi/v/datasette-hovercards.svg)](https://pypi.org/project/datasette-hovercards/)
  [![Changelog](https://img.shields.io/github/v/release/simonw/datasette-hovercards?include_prereleases&label=changelog)](https://github.com/simonw/datasette-hovercards/releases)
  [![Tests](https://github.com/simonw/datasette-hovercards/workflows/Test/badge.svg)](https://github.com/simonw/datasette-hovercards/actions?query=workflow%3ATest)
  [![License](https://img.shields.io/badge/license-Apache%202.0-blue.svg)](https://github.com/simonw/datasette-hovercards/blob/main/LICENSE)

  Add preview hovercards to links in Datasette

  ## Installation

  Install this plugin in the same environment as Datasette.

      $ datasette install datasette-hovercards

  ## Usage

  Once installed, hovering over a link to a row within the Datasette interface - for example a foreign key reference on the table page - should show a hovercard with a preview of that row.

  For a live demo, hover over values in the `user`, `milestone` or `repo` columns on this table page:

  https://latest-with-plugins.datasette.io/github/issues

  ## Development

  To set up this plugin locally, first checkout the code. Then create a new virtual environment:

      cd datasette-hovercards
      python3 -mvenv venv
      source venv/bin/activate

  Or if you are using `pipenv`:

      pipenv shell

  Now install the dependencies and test dependencies:

      pip install -e '.[test]'

  To run the tests:

      pytest

readme_html:
  <div id="readme" class="md" data-path="README.md"><article class="markdown-body entry-content container-lg" itemprop="text"><div class="markdown-heading" dir="auto"><h1 class="heading-element" dir="auto">datasette-hovercards</h1><a id="user-content-datasette-hovercards" class="anchor" aria-label="Permalink: datasette-hovercards" href="#datasette-hovercards"><svg class="octicon octicon-link" viewbox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
  <p dir="auto"><a href="https://pypi.org/project/datasette-hovercards/" rel="nofollow"><img src="https://img.shields.io/pypi/v/datasette-hovercards.svg" alt="PyPI" style="max-width: 100%;" data-camo-src="https://camo.githubusercontent.com/912a3aab0fdf93e620a9a0422eae33d08ec2444f02409a94fbaf775d1da8b371/68747470733a2f2f696d672e736869656c64732e696f2f707970692f762f6461746173657474652d686f76657263617264732e737667"></a>
  <a href="https://github.com/simonw/datasette-hovercards/releases"><img src="https://img.shields.io/github/v/release/simonw/datasette-hovercards?include_prereleases&label=changelog" alt="Changelog" style="max-width: 100%;" data-camo-src="https://camo.githubusercontent.com/ae35afaf5e77eb7d02e3e6bb05439f1f8539c0a0c77d46939777bff3a887450f/68747470733a2f2f696d672e736869656c64732e696f2f6769746875622f762f72656c656173652f73696d6f6e772f6461746173657474652d686f76657263617264733f696e636c7564655f70726572656c6561736573266c6162656c3d6368616e67656c6f67"></a>
  <a href="https://github.com/simonw/datasette-hovercards/actions?query=workflow%3ATest"><img src="https://github.com/simonw/datasette-hovercards/workflows/Test/badge.svg" alt="Tests" style="max-width: 100%;"></a>
  <a href="https://github.com/simonw/datasette-hovercards/blob/main/LICENSE"><img src="https://img.shields.io/badge/license-Apache%202.0-blue.svg" alt="License" style="max-width: 100%;" data-camo-src="https://camo.githubusercontent.com/c355f200ea90fddaa407b6eaab303663a669248ea3ca7b1fcf77dbe04ff5f48c/68747470733a2f2f696d672e736869656c64732e696f2f62616467652f6c6963656e73652d417061636865253230322e302d626c75652e737667"></a></p>
  <p dir="auto">Add preview hovercards to links in Datasette</p>
  <div class="markdown-heading" dir="auto"><h2 class="heading-element" dir="auto">Installation</h2><a id="user-content-installation" class="anchor" aria-label="Permalink: Installation" href="#installation"><svg class="octicon octicon-link" viewbox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
  <p dir="auto">Install this plugin in the same environment as Datasette.</p>
  <div class="snippet-clipboard-content notranslate position-relative overflow-auto" data-snippet-clipboard-copy-content="$ datasette install datasette-hovercards"><pre class="notranslate"><code>$ datasette install datasette-hovercards
  </code></pre></div>
  <div class="markdown-heading" dir="auto"><h2 class="heading-element" dir="auto">Usage</h2><a id="user-content-usage" class="anchor" aria-label="Permalink: Usage" href="#usage"><svg class="octicon octicon-link" viewbox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
  <p dir="auto">Once installed, hovering over a link to a row within the Datasette interface - for example a foreign key reference on the table page - should show a hovercard with a preview of that row.</p>
  <p dir="auto">For a live demo, hover over values in the <code>user</code>, <code>milestone</code> or <code>repo</code> columns on this table page:</p>
  <p dir="auto"><a href="https://latest-with-plugins.datasette.io/github/issues" rel="nofollow">https://latest-with-plugins.datasette.io/github/issues</a></p>
  <div class="markdown-heading" dir="auto"><h2 class="heading-element" dir="auto">Development</h2><a id="user-content-development" class="anchor" aria-label="Permalink: Development" href="#development"><svg class="octicon octicon-link" viewbox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
  <p dir="auto">To set up this plugin locally, first checkout the code. Then create a new virtual environment:</p>
  <div class="snippet-clipboard-content notranslate position-relative overflow-auto" data-snippet-clipboard-copy-content="cd datasette-hovercards
  python3 -mvenv venv
  source venv/bin/activate"><pre class="notranslate"><code>cd datasette-hovercards
  python3 -mvenv venv
  source venv/bin/activate
  </code></pre></div>
  <p dir="auto">Or if you are using <code>pipenv</code>:</p>
  <div class="snippet-clipboard-content notranslate position-relative overflow-auto" data-snippet-clipboard-copy-content="pipenv shell"><pre class="notranslate"><code>pipenv shell
  </code></pre></div>
  <p dir="auto">Now install the dependencies and test dependencies:</p>
  <div class="snippet-clipboard-content notranslate position-relative overflow-auto" data-snippet-clipboard-copy-content="pip install -e '.[test]'"><pre class="notranslate"><code>pip install -e '.[test]'
  </code></pre></div>
  <p dir="auto">To run the tests:</p>
  <div class="snippet-clipboard-content notranslate position-relative overflow-auto" data-snippet-clipboard-copy-content="pytest"><pre class="notranslate"><code>pytest
  </code></pre></div>
  </article></div>


node_id: R_kgDOGhtmrg
name: datasette-pretty-traces
full_name: simonw/datasette-pretty-traces
html_url: https://github.com/simonw/datasette-pretty-traces
description: Prettier formatting for ?_trace=1 traces
created_at: 2021-12-13T19:43:28Z
updated_at: 2026-02-08T23:09:34Z
pushed_at: 2026-02-08T23:09:30Z
language: JavaScript
license: apache-2.0
topics: ["datasette", "datasette-io", "datasette-plugin"]
visibility: public
default_branch: main
permissions: {"admin": false, "maintain": false, "push": false, "triage": false, "pull": false}
pull_request_creation_policy: all
readme:
  # datasette-pretty-traces

  [![PyPI](https://img.shields.io/pypi/v/datasette-pretty-traces.svg)](https://pypi.org/project/datasette-pretty-traces/)
  [![Changelog](https://img.shields.io/github/v/release/simonw/datasette-pretty-traces?include_prereleases&label=changelog)](https://github.com/simonw/datasette-pretty-traces/releases)
  [![Tests](https://github.com/simonw/datasette-pretty-traces/workflows/Test/badge.svg)](https://github.com/simonw/datasette-pretty-traces/actions?query=workflow%3ATest)
  [![License](https://img.shields.io/badge/license-Apache%202.0-blue.svg)](https://github.com/simonw/datasette-pretty-traces/blob/main/LICENSE)

  Prettier formatting for `?_trace=1` traces

  ## Installation

  Install this plugin in the same environment as Datasette.

      $ datasette install datasette-pretty-traces

  ## Usage

  Once installed, run Datasette using `--setting trace_debug 1`:

      datasette fixtures.db --setting trace_debug 1

  Then navigate to any page and add `?_trace=` to the URL:

      http://localhost:8001/?_trace=1

  The plugin will scroll you down the page to the visualized trace information.

  ## Demo

  You can try out the demo here:

  - [/?_trace=1](https://latest-with-plugins.datasette.io/?_trace=1) tracing the homepage
  - [/github/commits?_trace=1](https://latest-with-plugins.datasette.io/github/commits?_trace=1) tracing a table page

  ## Screenshot

  ![Screenshot showing the visualization produced by the plugin](https://user-images.githubusercontent.com/9599/145883732-a53accdd-5feb-4629-94cd-f73407c7943d.png)

  ## Development

  To set up this plugin locally, first checkout the code. Then create a new virtual environment:

      cd datasette-pretty-traces
      python3 -mvenv venv
      source venv/bin/activate

  Or if you are using `pipenv`:

      pipenv shell

  Now install the dependencies and test dependencies:

      pip install -e '.[test]'

  To run the tests:

      pytest

readme_html:
  <div id="readme" class="md" data-path="README.md"><article class="markdown-body entry-content container-lg" itemprop="text"><div class="markdown-heading" dir="auto"><h1 class="heading-element" dir="auto">datasette-pretty-traces</h1><a id="user-content-datasette-pretty-traces" class="anchor" aria-label="Permalink: datasette-pretty-traces" href="#datasette-pretty-traces"><svg class="octicon octicon-link" viewbox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
  <p dir="auto"><a href="https://pypi.org/project/datasette-pretty-traces/" rel="nofollow"><img src="https://img.shields.io/pypi/v/datasette-pretty-traces.svg" alt="PyPI" style="max-width: 100%;" data-camo-src="https://camo.githubusercontent.com/5e232340336ec18d2e70fea11cc5691b9649dc12023d16629cb3d2cc2f618819/68747470733a2f2f696d672e736869656c64732e696f2f707970692f762f6461746173657474652d7072657474792d7472616365732e737667"></a>
  <a href="https://github.com/simonw/datasette-pretty-traces/releases"><img src="https://img.shields.io/github/v/release/simonw/datasette-pretty-traces?include_prereleases&label=changelog" alt="Changelog" style="max-width: 100%;" data-camo-src="https://camo.githubusercontent.com/405b424926c88aa2a8bbfe753868f3fac20f0582a85b63a462881762c259fb32/68747470733a2f2f696d672e736869656c64732e696f2f6769746875622f762f72656c656173652f73696d6f6e772f6461746173657474652d7072657474792d7472616365733f696e636c7564655f70726572656c6561736573266c6162656c3d6368616e67656c6f67"></a>
  <a href="https://github.com/simonw/datasette-pretty-traces/actions?query=workflow%3ATest"><img src="https://github.com/simonw/datasette-pretty-traces/workflows/Test/badge.svg" alt="Tests" style="max-width: 100%;"></a>
  <a href="https://github.com/simonw/datasette-pretty-traces/blob/main/LICENSE"><img src="https://img.shields.io/badge/license-Apache%202.0-blue.svg" alt="License" style="max-width: 100%;" data-camo-src="https://camo.githubusercontent.com/c355f200ea90fddaa407b6eaab303663a669248ea3ca7b1fcf77dbe04ff5f48c/68747470733a2f2f696d672e736869656c64732e696f2f62616467652f6c6963656e73652d417061636865253230322e302d626c75652e737667"></a></p>
  <p dir="auto">Prettier formatting for <code>?_trace=1</code> traces</p>
  <div class="markdown-heading" dir="auto"><h2 class="heading-element" dir="auto">Installation</h2><a id="user-content-installation" class="anchor" aria-label="Permalink: Installation" href="#installation"><svg class="octicon octicon-link" viewbox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
  <p dir="auto">Install this plugin in the same environment as Datasette.</p>
  <div class="snippet-clipboard-content notranslate position-relative overflow-auto" data-snippet-clipboard-copy-content="$ datasette install datasette-pretty-traces"><pre class="notranslate"><code>$ datasette install datasette-pretty-traces
  </code></pre></div>
  <div class="markdown-heading" dir="auto"><h2 class="heading-element" dir="auto">Usage</h2><a id="user-content-usage" class="anchor" aria-label="Permalink: Usage" href="#usage"><svg class="octicon octicon-link" viewbox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
  <p dir="auto">Once installed, run Datasette using <code>--setting trace_debug 1</code>:</p>
  <div class="snippet-clipboard-content notranslate position-relative overflow-auto" data-snippet-clipboard-copy-content="datasette fixtures.db --setting trace_debug 1"><pre class="notranslate"><code>datasette fixtures.db --setting trace_debug 1
  </code></pre></div>
  <p dir="auto">Then navigate to any page and add <code>?_trace=</code> to the URL:</p>
  <div class="snippet-clipboard-content notranslate position-relative overflow-auto" data-snippet-clipboard-copy-content="http://localhost:8001/?_trace=1"><pre class="notranslate"><code>http://localhost:8001/?_trace=1
  </code></pre></div>
  <p dir="auto">The plugin will scroll you down the page to the visualized trace information.</p>
  <div class="markdown-heading" dir="auto"><h2 class="heading-element" dir="auto">Demo</h2><a id="user-content-demo" class="anchor" aria-label="Permalink: Demo" href="#demo"><svg class="octicon octicon-link" viewbox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
  <p dir="auto">You can try out the demo here:</p>
  <ul dir="auto">
  <li><a href="https://latest-with-plugins.datasette.io/?_trace=1" rel="nofollow">/?_trace=1</a> tracing the homepage</li>
  <li><a href="https://latest-with-plugins.datasette.io/github/commits?_trace=1" rel="nofollow">/github/commits?_trace=1</a> tracing a table page</li>
  </ul>
  <div class="markdown-heading" dir="auto"><h2 class="heading-element" dir="auto">Screenshot</h2><a id="user-content-screenshot" class="anchor" aria-label="Permalink: Screenshot" href="#screenshot"><svg class="octicon octicon-link" viewbox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
  <p dir="auto"><a target="_blank" rel="noopener noreferrer nofollow" href="https://user-images.githubusercontent.com/9599/145883732-a53accdd-5feb-4629-94cd-f73407c7943d.png"><img src="https://user-images.githubusercontent.com/9599/145883732-a53accdd-5feb-4629-94cd-f73407c7943d.png" alt="Screenshot showing the visualization produced by the plugin" style="max-width: 100%;"></a></p>
  <div class="markdown-heading" dir="auto"><h2 class="heading-element" dir="auto">Development</h2><a id="user-content-development" class="anchor" aria-label="Permalink: Development" href="#development"><svg class="octicon octicon-link" viewbox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
  <p dir="auto">To set up this plugin locally, first checkout the code. Then create a new virtual environment:</p>
  <div class="snippet-clipboard-content notranslate position-relative overflow-auto" data-snippet-clipboard-copy-content="cd datasette-pretty-traces
  python3 -mvenv venv
  source venv/bin/activate"><pre class="notranslate"><code>cd datasette-pretty-traces
  python3 -mvenv venv
  source venv/bin/activate
  </code></pre></div>
  <p dir="auto">Or if you are using <code>pipenv</code>:</p>
  <div class="snippet-clipboard-content notranslate position-relative overflow-auto" data-snippet-clipboard-copy-content="pipenv shell"><pre class="notranslate"><code>pipenv shell
  </code></pre></div>
  <p dir="auto">Now install the dependencies and test dependencies:</p>
  <div class="snippet-clipboard-content notranslate position-relative overflow-auto" data-snippet-clipboard-copy-content="pip install -e '.[test]'"><pre class="notranslate"><code>pip install -e '.[test]'
  </code></pre></div>
  <p dir="auto">To run the tests:</p>
  <div class="snippet-clipboard-content notranslate position-relative overflow-auto" data-snippet-clipboard-copy-content="pytest"><pre class="notranslate"><code>pytest
  </code></pre></div>
  </article></div>


node_id: R_kgDOGkmBIg
name: datasette-tiddlywiki
full_name: simonw/datasette-tiddlywiki
html_url: https://github.com/simonw/datasette-tiddlywiki
description: Run TiddlyWiki in Datasette and save Tiddlers to a SQLite database
created_at: 2021-12-23T01:05:56Z
updated_at: 2025-01-04T21:16:15Z
pushed_at: 2022-03-08T01:36:10Z
language: HTML
license: apache-2.0
topics: []
visibility: public
default_branch: main
permissions: {"admin": false, "maintain": false, "push": false, "triage": false, "pull": true}
readme:
  # datasette-tiddlywiki

  [![PyPI](https://img.shields.io/pypi/v/datasette-tiddlywiki.svg)](https://pypi.org/project/datasette-tiddlywiki/)
  [![Changelog](https://img.shields.io/github/v/release/simonw/datasette-tiddlywiki?include_prereleases&label=changelog)](https://github.com/simonw/datasette-tiddlywiki/releases)
  [![Tests](https://github.com/simonw/datasette-tiddlywiki/workflows/Test/badge.svg)](https://github.com/simonw/datasette-tiddlywiki/actions?query=workflow%3ATest)
  [![License](https://img.shields.io/badge/license-Apache%202.0-blue.svg)](https://github.com/simonw/datasette-tiddlywiki/blob/main/LICENSE)

  Run [TiddlyWiki](https://tiddlywiki.com/) in Datasette and save Tiddlers to a SQLite database

  Read more about this project [on my blog](https://simonwillison.net/2021/Dec/24/datasette-tiddlywiki/).

  ## Installation

  Install this plugin in the same environment as Datasette.

      $ datasette install datasette-tiddlywiki

  ## Usage

  Start Datasette with a `tiddlywiki.db` database. You can create it if it does not yet exist using `--create`.

  You need to be signed in as the `root` user to write to the wiki, so use the `--root` option and click on the link it provides:

      % datasette tiddlywiki.db --create --root
      http://127.0.0.1:8001/-/auth-token?token=456670f1e8d01a8a33b71e17653130de17387336e29afcdfb4ab3d18261e6630
      # ...

  Navigate to `/-/tiddlywiki` on your instance to interact with TiddlyWiki.

  ## Authentication and permissions

  By default, the wiki can be read by anyone who has permission to read the `tiddlywiki.db` database. Only the signed in `root` user can write to it.

  You can sign in using the `--root` option described above, or you can set a password for that user using the [datasette-auth-passwords](https://datasette.io/plugins/datasette-auth-passwords) plugin and sign in using the `/-/login` page.

  You can use the `edit-tiddlywiki` permission to grant edit permisions to other users, using another plugin such as [datasette-permissions-sql](https://datasette.io/plugins/datasette-permissions-sql).

  You can use the `view-database` permission against the `tiddlywiki` database to control who can view the wiki.

  Datasette's permissions mechanism is described in full in [the Datasette documentation](https://docs.datasette.io/en/stable/authentication.html).

  ## Development

  To set up this plugin locally, first checkout the code. Then create a new virtual environment:

      cd datasette-tiddlywiki
      python3 -mvenv venv
      source venv/bin/activate

  Or if you are using `pipenv`:

      pipenv shell

  Now install the dependencies and test dependencies:

      pip install -e '.[test]'

  To run the tests:

      pytest

readme_html:
  <div id="readme" class="md" data-path="README.md"><article class="markdown-body entry-content container-lg" itemprop="text"><div class="markdown-heading" dir="auto"><h1 class="heading-element" dir="auto">datasette-tiddlywiki</h1><a id="user-content-datasette-tiddlywiki" class="anchor" aria-label="Permalink: datasette-tiddlywiki" href="#datasette-tiddlywiki"><svg class="octicon octicon-link" viewbox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
  <p dir="auto"><a href="https://pypi.org/project/datasette-tiddlywiki/" rel="nofollow"><img src="https://img.shields.io/pypi/v/datasette-tiddlywiki.svg" alt="PyPI" style="max-width: 100%;" data-camo-src="https://camo.githubusercontent.com/fb9c1c9a7bbf8aedbabee12ae2b4d858e11fc7bca4598cf284c451dea174db73/68747470733a2f2f696d672e736869656c64732e696f2f707970692f762f6461746173657474652d746964646c7977696b692e737667"></a>
  <a href="https://github.com/simonw/datasette-tiddlywiki/releases"><img src="https://img.shields.io/github/v/release/simonw/datasette-tiddlywiki?include_prereleases&label=changelog" alt="Changelog" style="max-width: 100%;" data-camo-src="https://camo.githubusercontent.com/5be96d0b09c2328dbda65802b475981cecb70f1b9ec8d300afd7ce1f3a69e552/68747470733a2f2f696d672e736869656c64732e696f2f6769746875622f762f72656c656173652f73696d6f6e772f6461746173657474652d746964646c7977696b693f696e636c7564655f70726572656c6561736573266c6162656c3d6368616e67656c6f67"></a>
  <a href="https://github.com/simonw/datasette-tiddlywiki/actions?query=workflow%3ATest"><img src="https://github.com/simonw/datasette-tiddlywiki/workflows/Test/badge.svg" alt="Tests" style="max-width: 100%;"></a>
  <a href="https://github.com/simonw/datasette-tiddlywiki/blob/main/LICENSE"><img src="https://img.shields.io/badge/license-Apache%202.0-blue.svg" alt="License" style="max-width: 100%;" data-camo-src="https://camo.githubusercontent.com/c355f200ea90fddaa407b6eaab303663a669248ea3ca7b1fcf77dbe04ff5f48c/68747470733a2f2f696d672e736869656c64732e696f2f62616467652f6c6963656e73652d417061636865253230322e302d626c75652e737667"></a></p>
  <p dir="auto">Run <a href="https://tiddlywiki.com/" rel="nofollow">TiddlyWiki</a> in Datasette and save Tiddlers to a SQLite database</p>
  <p dir="auto">Read more about this project <a href="https://simonwillison.net/2021/Dec/24/datasette-tiddlywiki/" rel="nofollow">on my blog</a>.</p>
  <div class="markdown-heading" dir="auto"><h2 class="heading-element" dir="auto">Installation</h2><a id="user-content-installation" class="anchor" aria-label="Permalink: Installation" href="#installation"><svg class="octicon octicon-link" viewbox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
  <p dir="auto">Install this plugin in the same environment as Datasette.</p>
  <div class="snippet-clipboard-content notranslate position-relative overflow-auto" data-snippet-clipboard-copy-content="$ datasette install datasette-tiddlywiki"><pre class="notranslate"><code>$ datasette install datasette-tiddlywiki
  </code></pre></div>
  <div class="markdown-heading" dir="auto"><h2 class="heading-element" dir="auto">Usage</h2><a id="user-content-usage" class="anchor" aria-label="Permalink: Usage" href="#usage"><svg class="octicon octicon-link" viewbox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
  <p dir="auto">Start Datasette with a <code>tiddlywiki.db</code> database. You can create it if it does not yet exist using <code>--create</code>.</p>
  <p dir="auto">You need to be signed in as the <code>root</code> user to write to the wiki, so use the <code>--root</code> option and click on the link it provides:</p>
  <div class="snippet-clipboard-content notranslate position-relative overflow-auto" data-snippet-clipboard-copy-content="% datasette tiddlywiki.db --create --root
  http://127.0.0.1:8001/-/auth-token?token=456670f1e8d01a8a33b71e17653130de17387336e29afcdfb4ab3d18261e6630
  # ..."><pre class="notranslate"><code>% datasette tiddlywiki.db --create --root
  http://127.0.0.1:8001/-/auth-token?token=456670f1e8d01a8a33b71e17653130de17387336e29afcdfb4ab3d18261e6630
  # ...
  </code></pre></div>
  <p dir="auto">Navigate to <code>/-/tiddlywiki</code> on your instance to interact with TiddlyWiki.</p>
  <div class="markdown-heading" dir="auto"><h2 class="heading-element" dir="auto">Authentication and permissions</h2><a id="user-content-authentication-and-permissions" class="anchor" aria-label="Permalink: Authentication and permissions" href="#authentication-and-permissions"><svg class="octicon octicon-link" viewbox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
  <p dir="auto">By default, the wiki can be read by anyone who has permission to read the <code>tiddlywiki.db</code> database. Only the signed in <code>root</code> user can write to it.</p>
  <p dir="auto">You can sign in using the <code>--root</code> option described above, or you can set a password for that user using the <a href="https://datasette.io/plugins/datasette-auth-passwords" rel="nofollow">datasette-auth-passwords</a> plugin and sign in using the <code>/-/login</code> page.</p>
  <p dir="auto">You can use the <code>edit-tiddlywiki</code> permission to grant edit permisions to other users, using another plugin such as <a href="https://datasette.io/plugins/datasette-permissions-sql" rel="nofollow">datasette-permissions-sql</a>.</p>
  <p dir="auto">You can use the <code>view-database</code> permission against the <code>tiddlywiki</code> database to control who can view the wiki.</p>
  <p dir="auto">Datasette's permissions mechanism is described in full in <a href="https://docs.datasette.io/en/stable/authentication.html" rel="nofollow">the Datasette documentation</a>.</p>
  <div class="markdown-heading" dir="auto"><h2 class="heading-element" dir="auto">Development</h2><a id="user-content-development" class="anchor" aria-label="Permalink: Development" href="#development"><svg class="octicon octicon-link" viewbox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
  <p dir="auto">To set up this plugin locally, first checkout the code. Then create a new virtual environment:</p>
  <div class="snippet-clipboard-content notranslate position-relative overflow-auto" data-snippet-clipboard-copy-content="cd datasette-tiddlywiki
  python3 -mvenv venv
  source venv/bin/activate"><pre class="notranslate"><code>cd datasette-tiddlywiki
  python3 -mvenv venv
  source venv/bin/activate
  </code></pre></div>
  <p dir="auto">Or if you are using <code>pipenv</code>:</p>
  <div class="snippet-clipboard-content notranslate position-relative overflow-auto" data-snippet-clipboard-copy-content="pipenv shell"><pre class="notranslate"><code>pipenv shell
  </code></pre></div>
  <p dir="auto">Now install the dependencies and test dependencies:</p>
  <div class="snippet-clipboard-content notranslate position-relative overflow-auto" data-snippet-clipboard-copy-content="pip install -e '.[test]'"><pre class="notranslate"><code>pip install -e '.[test]'
  </code></pre></div>
  <p dir="auto">To run the tests:</p>
  <div class="snippet-clipboard-content notranslate position-relative overflow-auto" data-snippet-clipboard-copy-content="pytest"><pre class="notranslate"><code>pytest
  </code></pre></div>
  </article></div>


node_id: R_kgDOG0Ywag
name: datasette-enrichments
full_name: datasette/datasette-enrichments
html_url: https://github.com/datasette/datasette-enrichments
description: Tools for running enrichments against data stored in Datasette
created_at: 2022-02-10T01:13:30Z
updated_at: 2026-02-20T21:17:03Z
pushed_at: 2025-11-06T03:14:30Z
homepage: https://enrichments.datasette.io
language: Python
license: apache-2.0
topics: ["datasette", "datasette-plugin"]
visibility: public
default_branch: main
permissions: {"admin": false, "maintain": false, "push": false, "triage": false, "pull": false}
custom_properties: {}
pull_request_creation_policy: all
readme:
  # datasette-enrichments

  [![PyPI](https://img.shields.io/pypi/v/datasette-enrichments.svg)](https://pypi.org/project/datasette-enrichments/)
  [![Changelog](https://img.shields.io/github/v/release/simonw/datasette-enrichments?include_prereleases&label=changelog)](https://github.com/simonw/datasette-enrichments/releases)
  [![Tests](https://github.com/simonw/datasette-enrichments/workflows/Test/badge.svg)](https://github.com/simonw/datasette-enrichments/actions?query=workflow%3ATest)
  [![License](https://img.shields.io/badge/license-Apache%202.0-blue.svg)](https://github.com/simonw/datasette-enrichments/blob/main/LICENSE)

  Tools for running enrichments against data stored in Datasette

  Potential use-cases for enrichments include:

  - Geocoding an address and populating a latitude and longitude column
  - Executing a template to generate output based on the values in each row
  - Fetching data from a URL and populating a column with the result
  - Executing OCR against a linked image or PDF file

  Documentation for this plugin lives at **[enrichments.datasette.io](https://enrichments.datasette.io/)**.

  ## Development

  To set up this plugin locally, first checkout the code. Then create a new virtual environment:
  ```bash
  cd datasette-enrichments
  python3 -mvenv venv
  source venv/bin/activate
  ```
  Or if you are using `pipenv`:
  ```bash
  pipenv shell
  ```
  Now install the dependencies and test dependencies:
  ```bash
  pip install -e '.[test]'
  ```
  To run the tests:
  ```bash
  pytest
  ```

readme_html:
  <div id="readme" class="md" data-path="README.md"><article class="markdown-body entry-content container-lg" itemprop="text"><div class="markdown-heading" dir="auto"><h1 class="heading-element" dir="auto">datasette-enrichments</h1><a id="user-content-datasette-enrichments" class="anchor" aria-label="Permalink: datasette-enrichments" href="#datasette-enrichments"><svg class="octicon octicon-link" viewbox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
  <p dir="auto"><a href="https://pypi.org/project/datasette-enrichments/" rel="nofollow"><img src="https://img.shields.io/pypi/v/datasette-enrichments.svg" alt="PyPI" style="max-width: 100%;" data-camo-src="https://camo.githubusercontent.com/d1434e081621110a937cae8b46fad53ea90bc7accebe9bdbe4064aa9cb3f2361/68747470733a2f2f696d672e736869656c64732e696f2f707970692f762f6461746173657474652d656e726963686d656e74732e737667"></a>
  <a href="https://github.com/simonw/datasette-enrichments/releases"><img src="https://img.shields.io/github/v/release/simonw/datasette-enrichments?include_prereleases&label=changelog" alt="Changelog" style="max-width: 100%;" data-camo-src="https://camo.githubusercontent.com/44fe00944b8b6d55299705b83d99f46e97910f52a6d9d76780cac85d9a0367e0/68747470733a2f2f696d672e736869656c64732e696f2f6769746875622f762f72656c656173652f73696d6f6e772f6461746173657474652d656e726963686d656e74733f696e636c7564655f70726572656c6561736573266c6162656c3d6368616e67656c6f67"></a>
  <a href="https://github.com/simonw/datasette-enrichments/actions?query=workflow%3ATest"><img src="https://github.com/simonw/datasette-enrichments/workflows/Test/badge.svg" alt="Tests" style="max-width: 100%;"></a>
  <a href="https://github.com/simonw/datasette-enrichments/blob/main/LICENSE"><img src="https://img.shields.io/badge/license-Apache%202.0-blue.svg" alt="License" style="max-width: 100%;" data-camo-src="https://camo.githubusercontent.com/c355f200ea90fddaa407b6eaab303663a669248ea3ca7b1fcf77dbe04ff5f48c/68747470733a2f2f696d672e736869656c64732e696f2f62616467652f6c6963656e73652d417061636865253230322e302d626c75652e737667"></a></p>
  <p dir="auto">Tools for running enrichments against data stored in Datasette</p>
  <p dir="auto">Potential use-cases for enrichments include:</p>
  <ul dir="auto">
  <li>Geocoding an address and populating a latitude and longitude column</li>
  <li>Executing a template to generate output based on the values in each row</li>
  <li>Fetching data from a URL and populating a column with the result</li>
  <li>Executing OCR against a linked image or PDF file</li>
  </ul>
  <p dir="auto">Documentation for this plugin lives at <strong><a href="https://enrichments.datasette.io/" rel="nofollow">enrichments.datasette.io</a></strong>.</p>
  <div class="markdown-heading" dir="auto"><h2 class="heading-element" dir="auto">Development</h2><a id="user-content-development" class="anchor" aria-label="Permalink: Development" href="#development"><svg class="octicon octicon-link" viewbox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
  <p dir="auto">To set up this plugin locally, first checkout the code. Then create a new virtual environment:</p>
  <div class="highlight highlight-source-shell notranslate position-relative overflow-auto" dir="auto" data-snippet-clipboard-copy-content="cd datasette-enrichments
  python3 -mvenv venv
  source venv/bin/activate"><pre><span class="pl-c1">cd</span> datasette-enrichments
  python3 -mvenv venv
  <span class="pl-c1">source</span> venv/bin/activate</pre></div>
  <p dir="auto">Or if you are using <code>pipenv</code>:</p>
  <div class="highlight highlight-source-shell notranslate position-relative overflow-auto" dir="auto" data-snippet-clipboard-copy-content="pipenv shell"><pre>pipenv shell</pre></div>
  <p dir="auto">Now install the dependencies and test dependencies:</p>
  <div class="highlight highlight-source-shell notranslate position-relative overflow-auto" dir="auto" data-snippet-clipboard-copy-content="pip install -e '.[test]'"><pre>pip install -e <span class="pl-s"><span class="pl-pds">'</span>.[test]<span class="pl-pds">'</span></span></pre></div>
  <p dir="auto">To run the tests:</p>
  <div class="highlight highlight-source-shell notranslate position-relative overflow-auto" dir="auto" data-snippet-clipboard-copy-content="pytest"><pre>pytest</pre></div>
  </article></div>


node_id: R_kgDOG2hQNg
name: google-drive-to-sqlite
full_name: simonw/google-drive-to-sqlite
html_url: https://github.com/simonw/google-drive-to-sqlite
description: Create a SQLite database containing metadata from Google Drive
created_at: 2022-02-16T02:16:29Z
updated_at: 2025-05-03T09:27:54Z
pushed_at: 2025-03-12T02:58:46Z
homepage: https://datasette.io/tools/google-drive-to-sqlite
language: Python
license: apache-2.0
topics: []
visibility: public
default_branch: main
permissions: {"admin": false, "maintain": false, "push": false, "triage": false, "pull": true}
readme:
  # google-drive-to-sqlite

  [![PyPI](https://img.shields.io/pypi/v/google-drive-to-sqlite.svg)](https://pypi.org/project/google-drive-to-sqlite/)
  [![Changelog](https://img.shields.io/github/v/release/simonw/google-drive-to-sqlite?include_prereleases&label=changelog)](https://github.com/simonw/google-drive-to-sqlite/releases)
  [![Tests](https://github.com/simonw/google-drive-to-sqlite/workflows/Test/badge.svg)](https://github.com/simonw/google-drive-to-sqlite/actions?query=workflow%3ATest)
  [![License](https://img.shields.io/badge/license-Apache%202.0-blue.svg)](https://github.com/simonw/google-drive-to-sqlite/blob/master/LICENSE)

  > [!WARNING]  
  > This tool no longer works due to Google deprecating the authentication mechanism it uses. See [issue #40](https://github.com/simonw/google-drive-to-sqlite/issues/40).

  Create a SQLite database containing metadata from [Google Drive](https://www.google.com/drive)

  For background on this project, see [Google Drive to SQLite](https://simonwillison.net/2022/Feb/20/google-drive-to-sqlite/) on my blog.

  If you use Google Drive, and especially if you have shared drives with other people there's a good chance you have hundreds or even thousands of files that you may not be fully aware of.

  This tool can download metadata about those files - their names, sizes, folders, content types, permissions, creation dates and more - and store them in a SQLite database.

  This lets you use SQL to analyze your Google Drive contents, using [Datasette](https://datasette.io/) or the SQLite command-line tool or any other SQLite database browsing software.

  ## Installation

  Install this tool using `pip`:

      pip install google-drive-to-sqlite

  ## Quickstart

  Authenticate with Google Drive by running:

      google-drive-to-sqlite auth

  Now create a SQLite database with metadata about all of the files you have starred using:

      google-drive-to-sqlite files starred.db --starred

  You can explore the resulting database using [Datasette](https://datasette.io/):

      $ pip install datasette
      $ datasette starred.db
      INFO:     Started server process [24661]
      INFO:     Uvicorn running on http://127.0.0.1:8001

  ## Authentication

  > :warning: **This application has not yet been verified by Google** - you may find you are unable to authenticate until that verification is complete. [#10](https://github.com/simonw/google-drive-to-sqlite/issues/10)
  >
  > You can work around this issue by [creating your own OAuth client ID key](https://til.simonwillison.net/googlecloud/google-oauth-cli-application) and passing it to the `auth` command using `--google-client-id` and `--google-client-secret`.

  First, authenticate with Google Drive using the `auth` command:

      $ google-drive-to-sqlite auth
      Visit the following URL to authenticate with Google Drive

      https://accounts.google.com/o/oauth2/v2/auth?...

      Then return here and paste in the resulting code:
      Paste code here: 

  Follow the link, sign in with Google Drive and then copy and paste the resulting code back into the tool.

  This will save an authentication token to the file called `auth.json` in the current directory.

  To specify a different location for that file, use the `--auth` option:

      google-drive-to-sqlite auth --auth ~/google-drive-auth.json

  The `auth` command also provides options for using a different scope, Google client ID and Google client secret. You can use these to create your own custom authentication tokens that can work with other Google APIs, see [issue #5](https://github.com/simonw/google-drive-to-sqlite/issues/5) for details.

  Full `--help`:

  <!-- [[[cog
  import cog
  from google_drive_to_sqlite import cli
  from click.testing import CliRunner
  runner = CliRunner()
  result = runner.invoke(cli.cli, ["auth", "--help"])
  help = result.output.replace("Usage: cli", "Usage: google-drive-to-sqlite")
  cog.out(
      "```\n{}\n```\n".format(help)
  )
  ]]] -->
  ```
  Usage: google-drive-to-sqlite auth [OPTIONS]

    Authenticate user and save credentials

  Options:
    -a, --auth FILE              Path to save token, defaults to auth.json
    --google-client-id TEXT      Custom Google client ID
    --google-client-secret TEXT  Custom Google client secret
    --scope TEXT                 Custom token scope
    --help                       Show this message and exit.

  ```
  <!-- [[[end]]] -->

  To revoke the token that is stored in `auth.json`, such that it cannot be used to access Google Drive in the future, run the `revoke` command:

      google-drive-to-sqlite revoke

  Or if your token is stored in another location:

      google-drive-to-sqlite revoke -a ~/google-drive-auth.json

  You will need to obtain a fresh token using the `auth` command in order to continue using this tool.

  ## google-drive-to-sqlite files

  To retrieve metadata about the files in your Google Drive, or a folder or search within it, use the `google-drive-to-sqlite files` command.

  This will default to writing details about every file in your Google Drive to a SQLite database:

      google-drive-to-sqlite files files.db

  Files and folders will be written to databases tables, which will be created if they do not yet exist. The database schema is [shown below](#database-schema).

  If a file or folder already exists, based on a matching `id`, it will be replaced with fresh data.

  Instead of writing to SQLite you can use `--json` to output as JSON, or `--nl` to output as newline-delimited JSON:

      google-drive-to-sqlite files --nl

  Use `--folder ID` to retrieve everything in a specified folder and its sub-folders:

      google-drive-to-sqlite files files.db --folder 1E6Zg2X2bjjtPzVfX8YqdXZDCoB3AVA7i

  Use `--q QUERY` to use a [custom search query](https://developers.google.com/drive/api/v3/reference/query-ref):

      google-drive-to-sqlite files files.db -q "viewedByMeTime > '2022-01-01'"

  The following shortcut options help build queries:

  - `--full-text TEXT` to search for files where the full text matches a search term
  - `--starred` for files and folders you have starred
  - `--trashed` for files and folders in the trash
  - `--shared-with-me` for files and folders that have been shared with you
  - `--apps` for Google Apps documents, spreadsheets, presentations and drawings (equivalent to setting all  of the next four options)
  - `--docs` for Google Apps documents
  - `--sheets` for Google Apps spreadsheets
  - `--presentations` for Google Apps presentations
  - `--drawings` for Google Apps drawings

  You can combine these - for example, this returns all files that you have starred and that were shared with you:

      google-drive-to-sqlite files highlights.db \
        --starred --shared-with-me

  Multiple options are treated as AND, with the exception of the Google Apps options which are treated as OR - so the following would retrieve all spreadsheets and presentations that have also been starred:

      google-drive-to-sqlite files highlights.db \
        --starred --sheets --presentations

  You can use `--stop-after X` to stop after retrieving X files, useful for trying out a new search pattern and seeing results straight away.

  The `--import-json` and `--import-nl` options are mainly useful for testing and developing this tool. They allow you to replay the JSON or newline-delimited JSON that was previously fetched using `--json` or `--nl` and use it to create a fresh SQLite database, without needing to make any outbound API calls:

      # Fetch all starred files from the API, write to starred.json
      google-drive-to-sqlite files -q 'starred = true' --json > starred.json
      # Now import that data into a new SQLite database file
      google-drive-to-sqlite files starred.db --import-json starred.json

  Full `--help`:

  <!-- [[[cog
  result = runner.invoke(cli.cli, ["files", "--help"])
  help = result.output.replace("Usage: cli", "Usage: google-drive-to-sqlite")
  cog.out(
      "```\n{}\n```\n".format(help)
  )
  ]]] -->
  ```
  Usage: google-drive-to-sqlite files [OPTIONS] [DATABASE]

    Retrieve metadata for files in Google Drive, and write to a SQLite database or
    output as JSON.

        google-drive-to-sqlite files files.db

    Use --json to output JSON, --nl for newline-delimited JSON:

        google-drive-to-sqlite files files.db --json

    Use a folder ID to recursively fetch every file in that folder and its sub-
    folders:

        google-drive-to-sqlite files files.db --folder
        1E6Zg2X2bjjtPzVfX8YqdXZDCoB3AVA7i

    Fetch files you have starred:

        google-drive-to-sqlite files starred.db --starred

  Options:
    -a, --auth FILE       Path to auth.json token file
    --folder TEXT         Files in this folder ID and its sub-folders
    -q TEXT               Files matching this query
    --full-text TEXT      Search for files with text match
    --starred             Files you have starred
    --trashed             Files in the trash
    --shared-with-me      Files that have been shared with you
    --apps                Google Apps docs, spreadsheets, presentations and
                          drawings
    --docs                Google Apps docs
    --sheets              Google Apps spreadsheets
    --presentations       Google Apps presentations
    --drawings            Google Apps drawings
    --json                Output JSON rather than write to DB
    --nl                  Output newline-delimited JSON rather than write to DB
    --stop-after INTEGER  Stop paginating after X results
    --import-json FILE    Import from this JSON file instead of the API
    --import-nl FILE      Import from this newline-delimited JSON file
    -v, --verbose         Send verbose output to stderr
    --help                Show this message and exit.

  ```
  <!-- [[[end]]] -->

  ## google-drive-to-sqlite download FILE_ID

  The `download` command can be used to download files from Google Drive.

  You'll need one or more file IDs, which look something like `0B32uDVNZfiEKLUtIT1gzYWN2NDI4SzVQYTFWWWxCWUtvVGNB`.

  To download the file, run this:

      google-drive-to-sqlite download 0B32uDVNZfiEKLUtIT1gzYWN2NDI4SzVQYTFWWWxCWUtvVGNB

  This will detect the content type of the file and use that as the extension - so if this file is a JPEG the file would be downloaded as:

      0B32uDVNZfiEKLUtIT1gzYWN2NDI4SzVQYTFWWWxCWUtvVGNB.jpeg

  You can pass multiple file IDs to the command at once.

  To hide the progress bar and filename output, use `-s` or `--silent`.

  If you are downloading a single file you can use the `-o` output to specify a filename and location:

      google-drive-to-sqlite download 0B32uDVNZfiEKLUtIT1gzYWN2NDI4SzVQYTFWWWxCWUtvVGNB \
        -o my-image.jpeg

  Use `-o -` to write the file contents to standard output:

      google-drive-to-sqlite download 0B32uDVNZfiEKLUtIT1gzYWN2NDI4SzVQYTFWWWxCWUtvVGNB \
        -o - > my-image.jpeg

  Full `--help`:

  <!-- [[[cog
  result = runner.invoke(cli.cli, ["download", "--help"])
  help = result.output.replace("Usage: cli", "Usage: google-drive-to-sqlite")
  cog.out(
      "```\n{}\n```\n".format(help)
  )
  ]]] -->
  ```
  Usage: google-drive-to-sqlite download [OPTIONS] FILE_IDS...

    Download one or more files to disk, based on their file IDs.

    The file content will be saved to a file with the name:

        FILE_ID.ext

    Where the extension is automatically picked based on the type of file.

    If you are downloading a single file you can specify a filename with -o:

        google-drive-to-sqlite download MY_FILE_ID -o myfile.txt

  Options:
    -a, --auth FILE    Path to auth.json token file
    -o, --output FILE  File to write to, or - for standard output
    -s, --silent       Hide progress bar and filename
    --help             Show this message and exit.

  ```
  <!-- [[[end]]] -->

  ## google-drive-to-sqlite export FORMAT FILE_ID

  The `export` command can be used to export Google Docs documents, spreadsheets and presentations in a number of different formats.

  You'll need one or more document IDs, which look something like `10BOHGDUYa7lBjUSo26YFCHTpgEmtXabdVFaopCTh1vU`. You can find these by looking at the URL of your document on the Google Docs site.

  To export that document as PDF, run this:

      google-drive-to-sqlite export pdf 10BOHGDUYa7lBjUSo26YFCHTpgEmtXabdVFaopCTh1vU

  The file will be exported as:

      10BOHGDUYa7lBjUSo26YFCHTpgEmtXabdVFaopCTh1vU-export.pdf

  You can pass multiple file IDs to the command at once.

  For the `FORMAT` option you can use any of the mime type options listed [on this page](https://developers.google.com/drive/api/v3/ref-export-formats) - for example, to export as an Open Office document you could use:

      google-drive-to-sqlite export \
       application/vnd.oasis.opendocument.text \
       10BOHGDUYa7lBjUSo26YFCHTpgEmtXabdVFaopCTh1vU

  For convenience the following shortcuts for common file formats are provided:

  - Google Docs: `html`, `txt`, `rtf`, `pdf`, `doc`, `zip`, `epub`
  - Google Sheets: `xls`, `pdf`, `csv`, `tsv`, `zip`
  - Presentations: `ppt`, `pdf`, `txt`
  - Drawings: `jpeg`, `png`, `svg`

  The `zip` option returns a zip file of HTML. `txt` returns plain text. The others should be self-evident.

  To hide the filename output, use `-s` or `--silent`.

  If you are exporting a single file you can use the `-o` output to specify a filename and location:

      google-drive-to-sqlite export pdf 10BOHGDUYa7lBjUSo26YFCHTpgEmtXabdVFaopCTh1vU \
        -o my-document.pdf

  Use `-o -` to write the file contents to standard output:

      google-drive-to-sqlite export pdf 10BOHGDUYa7lBjUSo26YFCHTpgEmtXabdVFaopCTh1vU \
        -o - > my-document.pdf

  Full `--help`:

  <!-- [[[cog
  result = runner.invoke(cli.cli, ["export", "--help"])
  help = result.output.replace("Usage: cli", "Usage: google-drive-to-sqlite")
  cog.out(
      "```\n{}\n```\n".format(help)
  )
  ]]] -->
  ```
  Usage: google-drive-to-sqlite export [OPTIONS] FORMAT FILE_IDS...

    Export one or more files to the specified format.

    Usage:

        google-drive-to-sqlite export pdf FILE_ID_1 FILE_ID_2

    The file content will be saved to a file with the name:

        FILE_ID-export.ext

    Where the extension is based on the format you specified.

    Available export formats can be seen here:
    https://developers.google.com/drive/api/v3/ref-export-formats

    Or you can use one of the following shortcuts:

    - Google Docs: html, txt, rtf, pdf, doc, zip, epub
    - Google Sheets: xls, pdf, csv, tsv, zip
    - Presentations: ppt, pdf, txt
    - Drawings: jpeg, png, svg

    "zip" returns a zip file of HTML.

    If you are exporting a single file you can specify a filename with -o:

        google-drive-to-sqlite export zip MY_FILE_ID -o myfile.zip

  Options:
    -a, --auth FILE    Path to auth.json token file
    -o, --output FILE  File to write to, or - for standard output
    -s, --silent       Hide progress bar and filename
    --help             Show this message and exit.

  ```
  <!-- [[[end]]] -->

  ## google-drive-to-sqlite get URL

  The `get` command makes authenticated requests to the specified URL, using credentials derived from the `auth.json` file.

  For example:

      $ google-drive-to-sqlite get 'https://www.googleapis.com/drive/v3/about?fields=*'
      {
          "kind": "drive#about",
          "user": {
              "kind": "drive#user",
              "displayName": "Simon Willison",
      # ...

  If the resource you are fetching supports pagination you can use `--paginate key` to paginate through all of the rows in a specified key. For example, the following API has a `nextPageToken` key and a `files` list, suggesting it supports pagination:

      $ google-drive-to-sqlite get https://www.googleapis.com/drive/v3/files
      {
          "kind": "drive#fileList",
          "nextPageToken": "~!!~AI9...wogHHYlc=",
          "incompleteSearch": false,
          "files": [
              {
                  "kind": "drive#file",
                  "id": "1YEsITp_X8PtDUJWHGM0osT-TXAU1nr0e7RSWRM2Jpyg",
                  "name": "Title of a spreadsheet",
                  "mimeType": "application/vnd.google-apps.spreadsheet"
              },

  To paginate through everything in the `files` list you would use `--paginate files` like this:

      $ google-drive-to-sqlite get https://www.googleapis.com/drive/v3/files --paginate files
      [
        {
          "kind": "drive#file",
          "id": "1YEsITp_X8PtDUJWHGM0osT-TXAU1nr0e7RSWRM2Jpyg",
          "name": "Title of a spreadsheet",
          "mimeType": "application/vnd.google-apps.spreadsheet"
        },
        # ...

  Add `--nl` to stream paginated data as newline-delimited JSON:

      $ google-drive-to-sqlite get https://www.googleapis.com/drive/v3/files --paginate files --nl
      {"kind": "drive#file", "id": "1YEsITp_X8PtDUJWHGM0osT-TXAU1nr0e7RSWRM2Jpyg", "name": "Title of a spreadsheet", "mimeType": "application/vnd.google-apps.spreadsheet"}
      {"kind": "drive#file", "id": "1E6Zg2X2bjjtPzVfX8YqdXZDCoB3AVA7i", "name": "Subfolder", "mimeType": "application/vnd.google-apps.folder"}

  Add `--stop-after 5` to stop after 5 records - useful for testing.

  Full `--help`:

  <!-- [[[cog
  result = runner.invoke(cli.cli, ["get", "--help"])
  help = result.output.replace("Usage: cli", "Usage: google-drive-to-sqlite")
  cog.out(
      "```\n{}\n```\n".format(help)
  )
  ]]] -->
  ```
  Usage: google-drive-to-sqlite get [OPTIONS] URL

    Make an authenticated HTTP GET to the specified URL

  Options:
    -a, --auth FILE       Path to auth.json token file
    --paginate TEXT       Paginate through all results in this key
    --nl                  Output paginated data as newline-delimited JSON
    --stop-after INTEGER  Stop paginating after X results
    -v, --verbose         Send verbose output to stderr
    --help                Show this message and exit.

  ```
  <!-- [[[end]]] -->


  ## Database schema

  The database created by this tool has the following schema:

  <!-- [[[cog
  import tempfile, pathlib, sqlite_utils
  tmpdir = pathlib.Path(tempfile.mkdtemp())
  db_path = str(tmpdir / "docs.db")
  result = runner.invoke(cli.cli, [
      "files", db_path, "--import-json", "tests/folder-and-children.json"
  ])
  cog.out("```sql\n")
  schema = sqlite_utils.Database(db_path).schema
  # Tidy up some formatting
  schema = schema.replace(", [", ",\n   [")
  schema = schema.replace("\n,\n", ",\n")
  schema = schema.replace("TEXT);", "TEXT\n);")
  cog.out(schema)
  cog.out("\n```")
  ]]] -->
  ```sql
  CREATE TABLE [drive_users] (
     [permissionId] TEXT PRIMARY KEY,
     [kind] TEXT,
     [displayName] TEXT,
     [photoLink] TEXT,
     [me] INTEGER,
     [emailAddress] TEXT
  );
  CREATE TABLE [drive_folders] (
     [id] TEXT PRIMARY KEY,
     [_parent] TEXT,
     [_owner] TEXT,
     [lastModifyingUser] TEXT,
     [kind] TEXT,
     [name] TEXT,
     [mimeType] TEXT,
     [starred] INTEGER,
     [trashed] INTEGER,
     [explicitlyTrashed] INTEGER,
     [parents] TEXT,
     [spaces] TEXT,
     [version] TEXT,
     [webViewLink] TEXT,
     [iconLink] TEXT,
     [hasThumbnail] INTEGER,
     [thumbnailVersion] TEXT,
     [viewedByMe] INTEGER,
     [createdTime] TEXT,
     [modifiedTime] TEXT,
     [modifiedByMe] INTEGER,
     [shared] INTEGER,
     [ownedByMe] INTEGER,
     [viewersCanCopyContent] INTEGER,
     [copyRequiresWriterPermission] INTEGER,
     [writersCanShare] INTEGER,
     [folderColorRgb] TEXT,
     [quotaBytesUsed] TEXT,
     [isAppAuthorized] INTEGER,
     [linkShareMetadata] TEXT,
     FOREIGN KEY([_parent]) REFERENCES [drive_folders]([id]),
     FOREIGN KEY([_owner]) REFERENCES [drive_users]([permissionId]),
     FOREIGN KEY([lastModifyingUser]) REFERENCES [drive_users]([permissionId])
  );
  CREATE TABLE [drive_files] (
     [id] TEXT PRIMARY KEY,
     [_parent] TEXT,
     [_owner] TEXT,
     [lastModifyingUser] TEXT,
     [kind] TEXT,
     [name] TEXT,
     [mimeType] TEXT,
     [starred] INTEGER,
     [trashed] INTEGER,
     [explicitlyTrashed] INTEGER,
     [parents] TEXT,
     [spaces] TEXT,
     [version] TEXT,
     [webViewLink] TEXT,
     [iconLink] TEXT,
     [hasThumbnail] INTEGER,
     [thumbnailVersion] TEXT,
     [viewedByMe] INTEGER,
     [createdTime] TEXT,
     [modifiedTime] TEXT,
     [modifiedByMe] INTEGER,
     [shared] INTEGER,
     [ownedByMe] INTEGER,
     [viewersCanCopyContent] INTEGER,
     [copyRequiresWriterPermission] INTEGER,
     [writersCanShare] INTEGER,
     [quotaBytesUsed] TEXT,
     [isAppAuthorized] INTEGER,
     [linkShareMetadata] TEXT,
     FOREIGN KEY([_parent]) REFERENCES [drive_folders]([id]),
     FOREIGN KEY([_owner]) REFERENCES [drive_users]([permissionId]),
     FOREIGN KEY([lastModifyingUser]) REFERENCES [drive_users]([permissionId])
  );
  ```
  <!-- [[[end]]] -->

  ## Thumbnails

  You can construct a thumbnail image for a known file ID using the following URL:

      https://drive.google.com/thumbnail?sz=w800-h800&id=FILE_ID

  Users who are signed into Google Drive and have permission to view a file will be redirected to a thumbnail version of that file. You can tweak the `w800` and `h800` parameters to request different thumbnail sizes.

  ## Privacy policy

  This tool requests access to your Google Drive account in order to retrieve metadata about your files there. It also offers a feature that can download the content of those files.

  The credentials used to access your account are stored in the `auth.json` file on your computer. The metadata and content retrieved from Google Drive is also stored only on your own personal computer.

  At no point do the developers of this tool gain access to any of your data.

  ## Development

  To contribute to this tool, first checkout the code. Then create a new virtual environment:

      cd google-drive-to-sqlite
      python -m venv venv
      source venv/bin/activate

  Or if you are using `pipenv`:

      pipenv shell

  Now install the dependencies and test dependencies:

      pip install -e '.[test]'

  To run the tests:

      pytest

readme_html:
  <div id="readme" class="md" data-path="README.md"><article class="markdown-body entry-content container-lg" itemprop="text"><div class="markdown-heading" dir="auto"><h1 class="heading-element" dir="auto">google-drive-to-sqlite</h1><a id="user-content-google-drive-to-sqlite" class="anchor" aria-label="Permalink: google-drive-to-sqlite" href="#google-drive-to-sqlite"><svg class="octicon octicon-link" viewbox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
  <p dir="auto"><a href="https://pypi.org/project/google-drive-to-sqlite/" rel="nofollow"><img src="https://img.shields.io/pypi/v/google-drive-to-sqlite.svg" alt="PyPI" style="max-width: 100%;" data-camo-src="https://camo.githubusercontent.com/8ee21e222f051306f829c7f41fd891a445ea0937ccd8fa817c0e632cff3c0c86/68747470733a2f2f696d672e736869656c64732e696f2f707970692f762f676f6f676c652d64726976652d746f2d73716c6974652e737667"></a>
  <a href="https://github.com/simonw/google-drive-to-sqlite/releases"><img src="https://img.shields.io/github/v/release/simonw/google-drive-to-sqlite?include_prereleases&label=changelog" alt="Changelog" style="max-width: 100%;" data-camo-src="https://camo.githubusercontent.com/4db1ce9065618086922b44e658789b765c3db86af388331376d0649511f9eda5/68747470733a2f2f696d672e736869656c64732e696f2f6769746875622f762f72656c656173652f73696d6f6e772f676f6f676c652d64726976652d746f2d73716c6974653f696e636c7564655f70726572656c6561736573266c6162656c3d6368616e67656c6f67"></a>
  <a href="https://github.com/simonw/google-drive-to-sqlite/actions?query=workflow%3ATest"><img src="https://github.com/simonw/google-drive-to-sqlite/workflows/Test/badge.svg" alt="Tests" style="max-width: 100%;"></a>
  <a href="https://github.com/simonw/google-drive-to-sqlite/blob/master/LICENSE"><img src="https://img.shields.io/badge/license-Apache%202.0-blue.svg" alt="License" style="max-width: 100%;" data-camo-src="https://camo.githubusercontent.com/c355f200ea90fddaa407b6eaab303663a669248ea3ca7b1fcf77dbe04ff5f48c/68747470733a2f2f696d672e736869656c64732e696f2f62616467652f6c6963656e73652d417061636865253230322e302d626c75652e737667"></a></p>
  <div class="markdown-alert markdown-alert-warning" dir="auto"><p class="markdown-alert-title" dir="auto"><svg class="octicon octicon-alert mr-2" viewbox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="M6.457 1.047c.659-1.234 2.427-1.234 3.086 0l6.082 11.378A1.75 1.75 0 0 1 14.082 15H1.918a1.75 1.75 0 0 1-1.543-2.575Zm1.763.707a.25.25 0 0 0-.44 0L1.698 13.132a.25.25 0 0 0 .22.368h12.164a.25.25 0 0 0 .22-.368Zm.53 3.996v2.5a.75.75 0 0 1-1.5 0v-2.5a.75.75 0 0 1 1.5 0ZM9 11a1 1 0 1 1-2 0 1 1 0 0 1 2 0Z"></path></svg>Warning</p><p dir="auto">This tool no longer works due to Google deprecating the authentication mechanism it uses. See <a href="https://github.com/simonw/google-drive-to-sqlite/issues/40" data-hovercard-type="issue" data-hovercard-url="/simonw/google-drive-to-sqlite/issues/40/hovercard">issue #40</a>.</p>
  </div>
  <p dir="auto">Create a SQLite database containing metadata from <a href="https://www.google.com/drive" rel="nofollow">Google Drive</a></p>
  <p dir="auto">For background on this project, see <a href="https://simonwillison.net/2022/Feb/20/google-drive-to-sqlite/" rel="nofollow">Google Drive to SQLite</a> on my blog.</p>
  <p dir="auto">If you use Google Drive, and especially if you have shared drives with other people there's a good chance you have hundreds or even thousands of files that you may not be fully aware of.</p>
  <p dir="auto">This tool can download metadata about those files - their names, sizes, folders, content types, permissions, creation dates and more - and store them in a SQLite database.</p>
  <p dir="auto">This lets you use SQL to analyze your Google Drive contents, using <a href="https://datasette.io/" rel="nofollow">Datasette</a> or the SQLite command-line tool or any other SQLite database browsing software.</p>
  <div class="markdown-heading" dir="auto"><h2 class="heading-element" dir="auto">Installation</h2><a id="user-content-installation" class="anchor" aria-label="Permalink: Installation" href="#installation"><svg class="octicon octicon-link" viewbox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
  <p dir="auto">Install this tool using <code>pip</code>:</p>
  <div class="snippet-clipboard-content notranslate position-relative overflow-auto" data-snippet-clipboard-copy-content="pip install google-drive-to-sqlite"><pre class="notranslate"><code>pip install google-drive-to-sqlite
  </code></pre></div>
  <div class="markdown-heading" dir="auto"><h2 class="heading-element" dir="auto">Quickstart</h2><a id="user-content-quickstart" class="anchor" aria-label="Permalink: Quickstart" href="#quickstart"><svg class="octicon octicon-link" viewbox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
  <p dir="auto">Authenticate with Google Drive by running:</p>
  <div class="snippet-clipboard-content notranslate position-relative overflow-auto" data-snippet-clipboard-copy-content="google-drive-to-sqlite auth"><pre class="notranslate"><code>google-drive-to-sqlite auth
  </code></pre></div>
  <p dir="auto">Now create a SQLite database with metadata about all of the files you have starred using:</p>
  <div class="snippet-clipboard-content notranslate position-relative overflow-auto" data-snippet-clipboard-copy-content="google-drive-to-sqlite files starred.db --starred"><pre class="notranslate"><code>google-drive-to-sqlite files starred.db --starred
  </code></pre></div>
  <p dir="auto">You can explore the resulting database using <a href="https://datasette.io/" rel="nofollow">Datasette</a>:</p>
  <div class="snippet-clipboard-content notranslate position-relative overflow-auto" data-snippet-clipboard-copy-content="$ pip install datasette
  $ datasette starred.db
  INFO:     Started server process [24661]
  INFO:     Uvicorn running on http://127.0.0.1:8001"><pre class="notranslate"><code>$ pip install datasette
  $ datasette starred.db
  INFO:     Started server process [24661]
  INFO:     Uvicorn running on http://127.0.0.1:8001
  </code></pre></div>
  <div class="markdown-heading" dir="auto"><h2 class="heading-element" dir="auto">Authentication</h2><a id="user-content-authentication" class="anchor" aria-label="Permalink: Authentication" href="#authentication"><svg class="octicon octicon-link" viewbox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
  <blockquote>
  <p dir="auto"><g-emoji class="g-emoji" alias="warning">⚠️</g-emoji> <strong>This application has not yet been verified by Google</strong> - you may find you are unable to authenticate until that verification is complete. <a href="https://github.com/simonw/google-drive-to-sqlite/issues/10" data-hovercard-type="issue" data-hovercard-url="/simonw/google-drive-to-sqlite/issues/10/hovercard">#10</a></p>
  <p dir="auto">You can work around this issue by <a href="https://til.simonwillison.net/googlecloud/google-oauth-cli-application" rel="nofollow">creating your own OAuth client ID key</a> and passing it to the <code>auth</code> command using <code>--google-client-id</code> and <code>--google-client-secret</code>.</p>
  </blockquote>
  <p dir="auto">First, authenticate with Google Drive using the <code>auth</code> command:</p>
  <div class="snippet-clipboard-content notranslate position-relative overflow-auto" data-snippet-clipboard-copy-content="$ google-drive-to-sqlite auth
  Visit the following URL to authenticate with Google Drive

  https://accounts.google.com/o/oauth2/v2/auth?...

  Then return here and paste in the resulting code:
  Paste code here: "><pre class="notranslate"><code>$ google-drive-to-sqlite auth
  Visit the following URL to authenticate with Google Drive

  https://accounts.google.com/o/oauth2/v2/auth?...

  Then return here and paste in the resulting code:
  Paste code here: 
  </code></pre></div>
  <p dir="auto">Follow the link, sign in with Google Drive and then copy and paste the resulting code back into the tool.</p>
  <p dir="auto">This will save an authentication token to the file called <code>auth.json</code> in the current directory.</p>
  <p dir="auto">To specify a different location for that file, use the <code>--auth</code> option:</p>
  <div class="snippet-clipboard-content notranslate position-relative overflow-auto" data-snippet-clipboard-copy-content="google-drive-to-sqlite auth --auth ~/google-drive-auth.json"><pre class="notranslate"><code>google-drive-to-sqlite auth --auth ~/google-drive-auth.json
  </code></pre></div>
  <p dir="auto">The <code>auth</code> command also provides options for using a different scope, Google client ID and Google client secret. You can use these to create your own custom authentication tokens that can work with other Google APIs, see <a href="https://github.com/simonw/google-drive-to-sqlite/issues/5" data-hovercard-type="issue" data-hovercard-url="/simonw/google-drive-to-sqlite/issues/5/hovercard">issue #5</a> for details.</p>
  <p dir="auto">Full <code>--help</code>:</p>

  <div class="snippet-clipboard-content notranslate position-relative overflow-auto" data-snippet-clipboard-copy-content="Usage: google-drive-to-sqlite auth [OPTIONS]

    Authenticate user and save credentials

  Options:
    -a, --auth FILE              Path to save token, defaults to auth.json
    --google-client-id TEXT      Custom Google client ID
    --google-client-secret TEXT  Custom Google client secret
    --scope TEXT                 Custom token scope
    --help                       Show this message and exit.
  "><pre class="notranslate"><code>Usage: google-drive-to-sqlite auth [OPTIONS]

    Authenticate user and save credentials

  Options:
    -a, --auth FILE              Path to save token, defaults to auth.json
    --google-client-id TEXT      Custom Google client ID
    --google-client-secret TEXT  Custom Google client secret
    --scope TEXT                 Custom token scope
    --help                       Show this message and exit.

  </code></pre></div>

  <p dir="auto">To revoke the token that is stored in <code>auth.json</code>, such that it cannot be used to access Google Drive in the future, run the <code>revoke</code> command:</p>
  <div class="snippet-clipboard-content notranslate position-relative overflow-auto" data-snippet-clipboard-copy-content="google-drive-to-sqlite revoke"><pre class="notranslate"><code>google-drive-to-sqlite revoke
  </code></pre></div>
  <p dir="auto">Or if your token is stored in another location:</p>
  <div class="snippet-clipboard-content notranslate position-relative overflow-auto" data-snippet-clipboard-copy-content="google-drive-to-sqlite revoke -a ~/google-drive-auth.json"><pre class="notranslate"><code>google-drive-to-sqlite revoke -a ~/google-drive-auth.json
  </code></pre></div>
  <p dir="auto">You will need to obtain a fresh token using the <code>auth</code> command in order to continue using this tool.</p>
  <div class="markdown-heading" dir="auto"><h2 class="heading-element" dir="auto">google-drive-to-sqlite files</h2><a id="user-content-google-drive-to-sqlite-files" class="anchor" aria-label="Permalink: google-drive-to-sqlite files" href="#google-drive-to-sqlite-files"><svg class="octicon octicon-link" viewbox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
  <p dir="auto">To retrieve metadata about the files in your Google Drive, or a folder or search within it, use the <code>google-drive-to-sqlite files</code> command.</p>
  <p dir="auto">This will default to writing details about every file in your Google Drive to a SQLite database:</p>
  <div class="snippet-clipboard-content notranslate position-relative overflow-auto" data-snippet-clipboard-copy-content="google-drive-to-sqlite files files.db"><pre class="notranslate"><code>google-drive-to-sqlite files files.db
  </code></pre></div>
  <p dir="auto">Files and folders will be written to databases tables, which will be created if they do not yet exist. The database schema is <a href="#database-schema">shown below</a>.</p>
  <p dir="auto">If a file or folder already exists, based on a matching <code>id</code>, it will be replaced with fresh data.</p>
  <p dir="auto">Instead of writing to SQLite you can use <code>--json</code> to output as JSON, or <code>--nl</code> to output as newline-delimited JSON:</p>
  <div class="snippet-clipboard-content notranslate position-relative overflow-auto" data-snippet-clipboard-copy-content="google-drive-to-sqlite files --nl"><pre class="notranslate"><code>google-drive-to-sqlite files --nl
  </code></pre></div>
  <p dir="auto">Use <code>--folder ID</code> to retrieve everything in a specified folder and its sub-folders:</p>
  <div class="snippet-clipboard-content notranslate position-relative overflow-auto" data-snippet-clipboard-copy-content="google-drive-to-sqlite files files.db --folder 1E6Zg2X2bjjtPzVfX8YqdXZDCoB3AVA7i"><pre class="notranslate"><code>google-drive-to-sqlite files files.db --folder 1E6Zg2X2bjjtPzVfX8YqdXZDCoB3AVA7i
  </code></pre></div>
  <p dir="auto">Use <code>--q QUERY</code> to use a <a href="https://developers.google.com/drive/api/v3/reference/query-ref" rel="nofollow">custom search query</a>:</p>
  <div class="snippet-clipboard-content notranslate position-relative overflow-auto" data-snippet-clipboard-copy-content="google-drive-to-sqlite files files.db -q "viewedByMeTime > '2022-01-01'""><pre class="notranslate"><code>google-drive-to-sqlite files files.db -q "viewedByMeTime > '2022-01-01'"
  </code></pre></div>
  <p dir="auto">The following shortcut options help build queries:</p>
  <ul dir="auto">
  <li><code>--full-text TEXT</code> to search for files where the full text matches a search term</li>
  <li><code>--starred</code> for files and folders you have starred</li>
  <li><code>--trashed</code> for files and folders in the trash</li>
  <li><code>--shared-with-me</code> for files and folders that have been shared with you</li>
  <li><code>--apps</code> for Google Apps documents, spreadsheets, presentations and drawings (equivalent to setting all  of the next four options)</li>
  <li><code>--docs</code> for Google Apps documents</li>
  <li><code>--sheets</code> for Google Apps spreadsheets</li>
  <li><code>--presentations</code> for Google Apps presentations</li>
  <li><code>--drawings</code> for Google Apps drawings</li>
  </ul>
  <p dir="auto">You can combine these - for example, this returns all files that you have starred and that were shared with you:</p>
  <div class="snippet-clipboard-content notranslate position-relative overflow-auto" data-snippet-clipboard-copy-content="google-drive-to-sqlite files highlights.db \
    --starred --shared-with-me"><pre class="notranslate"><code>google-drive-to-sqlite files highlights.db \
    --starred --shared-with-me
  </code></pre></div>
  <p dir="auto">Multiple options are treated as AND, with the exception of the Google Apps options which are treated as OR - so the following would retrieve all spreadsheets and presentations that have also been starred:</p>
  <div class="snippet-clipboard-content notranslate position-relative overflow-auto" data-snippet-clipboard-copy-content="google-drive-to-sqlite files highlights.db \
    --starred --sheets --presentations"><pre class="notranslate"><code>google-drive-to-sqlite files highlights.db \
    --starred --sheets --presentations
  </code></pre></div>
  <p dir="auto">You can use <code>--stop-after X</code> to stop after retrieving X files, useful for trying out a new search pattern and seeing results straight away.</p>
  <p dir="auto">The <code>--import-json</code> and <code>--import-nl</code> options are mainly useful for testing and developing this tool. They allow you to replay the JSON or newline-delimited JSON that was previously fetched using <code>--json</code> or <code>--nl</code> and use it to create a fresh SQLite database, without needing to make any outbound API calls:</p>
  <div class="snippet-clipboard-content notranslate position-relative overflow-auto" data-snippet-clipboard-copy-content="# Fetch all starred files from the API, write to starred.json
  google-drive-to-sqlite files -q 'starred = true' --json > starred.json
  # Now import that data into a new SQLite database file
  google-drive-to-sqlite files starred.db --import-json starred.json"><pre class="notranslate"><code># Fetch all starred files from the API, write to starred.json
  google-drive-to-sqlite files -q 'starred = true' --json > starred.json
  # Now import that data into a new SQLite database file
  google-drive-to-sqlite files starred.db --import-json starred.json
  </code></pre></div>
  <p dir="auto">Full <code>--help</code>:</p>

  <div class="snippet-clipboard-content notranslate position-relative overflow-auto" data-snippet-clipboard-copy-content="Usage: google-drive-to-sqlite files [OPTIONS] [DATABASE]

    Retrieve metadata for files in Google Drive, and write to a SQLite database or
    output as JSON.

        google-drive-to-sqlite files files.db

    Use --json to output JSON, --nl for newline-delimited JSON:

        google-drive-to-sqlite files files.db --json

    Use a folder ID to recursively fetch every file in that folder and its sub-
    folders:

        google-drive-to-sqlite files files.db --folder
        1E6Zg2X2bjjtPzVfX8YqdXZDCoB3AVA7i

    Fetch files you have starred:

        google-drive-to-sqlite files starred.db --starred

  Options:
    -a, --auth FILE       Path to auth.json token file
    --folder TEXT         Files in this folder ID and its sub-folders
    -q TEXT               Files matching this query
    --full-text TEXT      Search for files with text match
    --starred             Files you have starred
    --trashed             Files in the trash
    --shared-with-me      Files that have been shared with you
    --apps                Google Apps docs, spreadsheets, presentations and
                          drawings
    --docs                Google Apps docs
    --sheets              Google Apps spreadsheets
    --presentations       Google Apps presentations
    --drawings            Google Apps drawings
    --json                Output JSON rather than write to DB
    --nl                  Output newline-delimited JSON rather than write to DB
    --stop-after INTEGER  Stop paginating after X results
    --import-json FILE    Import from this JSON file instead of the API
    --import-nl FILE      Import from this newline-delimited JSON file
    -v, --verbose         Send verbose output to stderr
    --help                Show this message and exit.
  "><pre class="notranslate"><code>Usage: google-drive-to-sqlite files [OPTIONS] [DATABASE]

    Retrieve metadata for files in Google Drive, and write to a SQLite database or
    output as JSON.

        google-drive-to-sqlite files files.db

    Use --json to output JSON, --nl for newline-delimited JSON:

        google-drive-to-sqlite files files.db --json

    Use a folder ID to recursively fetch every file in that folder and its sub-
    folders:

        google-drive-to-sqlite files files.db --folder
        1E6Zg2X2bjjtPzVfX8YqdXZDCoB3AVA7i

    Fetch files you have starred:

        google-drive-to-sqlite files starred.db --starred

  Options:
    -a, --auth FILE       Path to auth.json token file
    --folder TEXT         Files in this folder ID and its sub-folders
    -q TEXT               Files matching this query
    --full-text TEXT      Search for files with text match
    --starred             Files you have starred
    --trashed             Files in the trash
    --shared-with-me      Files that have been shared with you
    --apps                Google Apps docs, spreadsheets, presentations and
                          drawings
    --docs                Google Apps docs
    --sheets              Google Apps spreadsheets
    --presentations       Google Apps presentations
    --drawings            Google Apps drawings
    --json                Output JSON rather than write to DB
    --nl                  Output newline-delimited JSON rather than write to DB
    --stop-after INTEGER  Stop paginating after X results
    --import-json FILE    Import from this JSON file instead of the API
    --import-nl FILE      Import from this newline-delimited JSON file
    -v, --verbose         Send verbose output to stderr
    --help                Show this message and exit.

  </code></pre></div>

  <div class="markdown-heading" dir="auto"><h2 class="heading-element" dir="auto">google-drive-to-sqlite download FILE_ID</h2><a id="user-content-google-drive-to-sqlite-download-file_id" class="anchor" aria-label="Permalink: google-drive-to-sqlite download FILE_ID" href="#google-drive-to-sqlite-download-file_id"><svg class="octicon octicon-link" viewbox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
  <p dir="auto">The <code>download</code> command can be used to download files from Google Drive.</p>
  <p dir="auto">You'll need one or more file IDs, which look something like <code>0B32uDVNZfiEKLUtIT1gzYWN2NDI4SzVQYTFWWWxCWUtvVGNB</code>.</p>
  <p dir="auto">To download the file, run this:</p>
  <div class="snippet-clipboard-content notranslate position-relative overflow-auto" data-snippet-clipboard-copy-content="google-drive-to-sqlite download 0B32uDVNZfiEKLUtIT1gzYWN2NDI4SzVQYTFWWWxCWUtvVGNB"><pre class="notranslate"><code>google-drive-to-sqlite download 0B32uDVNZfiEKLUtIT1gzYWN2NDI4SzVQYTFWWWxCWUtvVGNB
  </code></pre></div>
  <p dir="auto">This will detect the content type of the file and use that as the extension - so if this file is a JPEG the file would be downloaded as:</p>
  <div class="snippet-clipboard-content notranslate position-relative overflow-auto" data-snippet-clipboard-copy-content="0B32uDVNZfiEKLUtIT1gzYWN2NDI4SzVQYTFWWWxCWUtvVGNB.jpeg"><pre class="notranslate"><code>0B32uDVNZfiEKLUtIT1gzYWN2NDI4SzVQYTFWWWxCWUtvVGNB.jpeg
  </code></pre></div>
  <p dir="auto">You can pass multiple file IDs to the command at once.</p>
  <p dir="auto">To hide the progress bar and filename output, use <code>-s</code> or <code>--silent</code>.</p>
  <p dir="auto">If you are downloading a single file you can use the <code>-o</code> output to specify a filename and location:</p>
  <div class="snippet-clipboard-content notranslate position-relative overflow-auto" data-snippet-clipboard-copy-content="google-drive-to-sqlite download 0B32uDVNZfiEKLUtIT1gzYWN2NDI4SzVQYTFWWWxCWUtvVGNB \
    -o my-image.jpeg"><pre class="notranslate"><code>google-drive-to-sqlite download 0B32uDVNZfiEKLUtIT1gzYWN2NDI4SzVQYTFWWWxCWUtvVGNB \
    -o my-image.jpeg
  </code></pre></div>
  <p dir="auto">Use <code>-o -</code> to write the file contents to standard output:</p>
  <div class="snippet-clipboard-content notranslate position-relative overflow-auto" data-snippet-clipboard-copy-content="google-drive-to-sqlite download 0B32uDVNZfiEKLUtIT1gzYWN2NDI4SzVQYTFWWWxCWUtvVGNB \
    -o - > my-image.jpeg"><pre class="notranslate"><code>google-drive-to-sqlite download 0B32uDVNZfiEKLUtIT1gzYWN2NDI4SzVQYTFWWWxCWUtvVGNB \
    -o - > my-image.jpeg
  </code></pre></div>
  <p dir="auto">Full <code>--help</code>:</p>

  <div class="snippet-clipboard-content notranslate position-relative overflow-auto" data-snippet-clipboard-copy-content="Usage: google-drive-to-sqlite download [OPTIONS] FILE_IDS...

    Download one or more files to disk, based on their file IDs.

    The file content will be saved to a file with the name:

        FILE_ID.ext

    Where the extension is automatically picked based on the type of file.

    If you are downloading a single file you can specify a filename with -o:

        google-drive-to-sqlite download MY_FILE_ID -o myfile.txt

  Options:
    -a, --auth FILE    Path to auth.json token file
    -o, --output FILE  File to write to, or - for standard output
    -s, --silent       Hide progress bar and filename
    --help             Show this message and exit.
  "><pre class="notranslate"><code>Usage: google-drive-to-sqlite download [OPTIONS] FILE_IDS...

    Download one or more files to disk, based on their file IDs.

    The file content will be saved to a file with the name:

        FILE_ID.ext

    Where the extension is automatically picked based on the type of file.

    If you are downloading a single file you can specify a filename with -o:

        google-drive-to-sqlite download MY_FILE_ID -o myfile.txt

  Options:
    -a, --auth FILE    Path to auth.json token file
    -o, --output FILE  File to write to, or - for standard output
    -s, --silent       Hide progress bar and filename
    --help             Show this message and exit.

  </code></pre></div>

  <div class="markdown-heading" dir="auto"><h2 class="heading-element" dir="auto">google-drive-to-sqlite export FORMAT FILE_ID</h2><a id="user-content-google-drive-to-sqlite-export-format-file_id" class="anchor" aria-label="Permalink: google-drive-to-sqlite export FORMAT FILE_ID" href="#google-drive-to-sqlite-export-format-file_id"><svg class="octicon octicon-link" viewbox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
  <p dir="auto">The <code>export</code> command can be used to export Google Docs documents, spreadsheets and presentations in a number of different formats.</p>
  <p dir="auto">You'll need one or more document IDs, which look something like <code>10BOHGDUYa7lBjUSo26YFCHTpgEmtXabdVFaopCTh1vU</code>. You can find these by looking at the URL of your document on the Google Docs site.</p>
  <p dir="auto">To export that document as PDF, run this:</p>
  <div class="snippet-clipboard-content notranslate position-relative overflow-auto" data-snippet-clipboard-copy-content="google-drive-to-sqlite export pdf 10BOHGDUYa7lBjUSo26YFCHTpgEmtXabdVFaopCTh1vU"><pre class="notranslate"><code>google-drive-to-sqlite export pdf 10BOHGDUYa7lBjUSo26YFCHTpgEmtXabdVFaopCTh1vU
  </code></pre></div>
  <p dir="auto">The file will be exported as:</p>
  <div class="snippet-clipboard-content notranslate position-relative overflow-auto" data-snippet-clipboard-copy-content="10BOHGDUYa7lBjUSo26YFCHTpgEmtXabdVFaopCTh1vU-export.pdf"><pre class="notranslate"><code>10BOHGDUYa7lBjUSo26YFCHTpgEmtXabdVFaopCTh1vU-export.pdf
  </code></pre></div>
  <p dir="auto">You can pass multiple file IDs to the command at once.</p>
  <p dir="auto">For the <code>FORMAT</code> option you can use any of the mime type options listed <a href="https://developers.google.com/drive/api/v3/ref-export-formats" rel="nofollow">on this page</a> - for example, to export as an Open Office document you could use:</p>
  <div class="snippet-clipboard-content notranslate position-relative overflow-auto" data-snippet-clipboard-copy-content="google-drive-to-sqlite export \
   application/vnd.oasis.opendocument.text \
   10BOHGDUYa7lBjUSo26YFCHTpgEmtXabdVFaopCTh1vU"><pre class="notranslate"><code>google-drive-to-sqlite export \
   application/vnd.oasis.opendocument.text \
   10BOHGDUYa7lBjUSo26YFCHTpgEmtXabdVFaopCTh1vU
  </code></pre></div>
  <p dir="auto">For convenience the following shortcuts for common file formats are provided:</p>
  <ul dir="auto">
  <li>Google Docs: <code>html</code>, <code>txt</code>, <code>rtf</code>, <code>pdf</code>, <code>doc</code>, <code>zip</code>, <code>epub</code></li>
  <li>Google Sheets: <code>xls</code>, <code>pdf</code>, <code>csv</code>, <code>tsv</code>, <code>zip</code></li>
  <li>Presentations: <code>ppt</code>, <code>pdf</code>, <code>txt</code></li>
  <li>Drawings: <code>jpeg</code>, <code>png</code>, <code>svg</code></li>
  </ul>
  <p dir="auto">The <code>zip</code> option returns a zip file of HTML. <code>txt</code> returns plain text. The others should be self-evident.</p>
  <p dir="auto">To hide the filename output, use <code>-s</code> or <code>--silent</code>.</p>
  <p dir="auto">If you are exporting a single file you can use the <code>-o</code> output to specify a filename and location:</p>
  <div class="snippet-clipboard-content notranslate position-relative overflow-auto" data-snippet-clipboard-copy-content="google-drive-to-sqlite export pdf 10BOHGDUYa7lBjUSo26YFCHTpgEmtXabdVFaopCTh1vU \
    -o my-document.pdf"><pre class="notranslate"><code>google-drive-to-sqlite export pdf 10BOHGDUYa7lBjUSo26YFCHTpgEmtXabdVFaopCTh1vU \
    -o my-document.pdf
  </code></pre></div>
  <p dir="auto">Use <code>-o -</code> to write the file contents to standard output:</p>
  <div class="snippet-clipboard-content notranslate position-relative overflow-auto" data-snippet-clipboard-copy-content="google-drive-to-sqlite export pdf 10BOHGDUYa7lBjUSo26YFCHTpgEmtXabdVFaopCTh1vU \
    -o - > my-document.pdf"><pre class="notranslate"><code>google-drive-to-sqlite export pdf 10BOHGDUYa7lBjUSo26YFCHTpgEmtXabdVFaopCTh1vU \
    -o - > my-document.pdf
  </code></pre></div>
  <p dir="auto">Full <code>--help</code>:</p>

  <div class="snippet-clipboard-content notranslate position-relative overflow-auto" data-snippet-clipboard-copy-content="Usage: google-drive-to-sqlite export [OPTIONS] FORMAT FILE_IDS...

    Export one or more files to the specified format.

    Usage:

        google-drive-to-sqlite export pdf FILE_ID_1 FILE_ID_2

    The file content will be saved to a file with the name:

        FILE_ID-export.ext

    Where the extension is based on the format you specified.

    Available export formats can be seen here:
    https://developers.google.com/drive/api/v3/ref-export-formats

    Or you can use one of the following shortcuts:

    - Google Docs: html, txt, rtf, pdf, doc, zip, epub
    - Google Sheets: xls, pdf, csv, tsv, zip
    - Presentations: ppt, pdf, txt
    - Drawings: jpeg, png, svg

    "zip" returns a zip file of HTML.

    If you are exporting a single file you can specify a filename with -o:

        google-drive-to-sqlite export zip MY_FILE_ID -o myfile.zip

  Options:
    -a, --auth FILE    Path to auth.json token file
    -o, --output FILE  File to write to, or - for standard output
    -s, --silent       Hide progress bar and filename
    --help             Show this message and exit.
  "><pre class="notranslate"><code>Usage: google-drive-to-sqlite export [OPTIONS] FORMAT FILE_IDS...

    Export one or more files to the specified format.

    Usage:

        google-drive-to-sqlite export pdf FILE_ID_1 FILE_ID_2

    The file content will be saved to a file with the name:

        FILE_ID-export.ext

    Where the extension is based on the format you specified.

    Available export formats can be seen here:
    https://developers.google.com/drive/api/v3/ref-export-formats

    Or you can use one of the following shortcuts:

    - Google Docs: html, txt, rtf, pdf, doc, zip, epub
    - Google Sheets: xls, pdf, csv, tsv, zip
    - Presentations: ppt, pdf, txt
    - Drawings: jpeg, png, svg

    "zip" returns a zip file of HTML.

    If you are exporting a single file you can specify a filename with -o:

        google-drive-to-sqlite export zip MY_FILE_ID -o myfile.zip

  Options:
    -a, --auth FILE    Path to auth.json token file
    -o, --output FILE  File to write to, or - for standard output
    -s, --silent       Hide progress bar and filename
    --help             Show this message and exit.

  </code></pre></div>

  <div class="markdown-heading" dir="auto"><h2 class="heading-element" dir="auto">google-drive-to-sqlite get URL</h2><a id="user-content-google-drive-to-sqlite-get-url" class="anchor" aria-label="Permalink: google-drive-to-sqlite get URL" href="#google-drive-to-sqlite-get-url"><svg class="octicon octicon-link" viewbox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
  <p dir="auto">The <code>get</code> command makes authenticated requests to the specified URL, using credentials derived from the <code>auth.json</code> file.</p>
  <p dir="auto">For example:</p>
  <div class="snippet-clipboard-content notranslate position-relative overflow-auto" data-snippet-clipboard-copy-content="$ google-drive-to-sqlite get 'https://www.googleapis.com/drive/v3/about?fields=*'
  {
      "kind": "drive#about",
      "user": {
          "kind": "drive#user",
          "displayName": "Simon Willison",
  # ..."><pre class="notranslate"><code>$ google-drive-to-sqlite get 'https://www.googleapis.com/drive/v3/about?fields=*'
  {
      "kind": "drive#about",
      "user": {
          "kind": "drive#user",
          "displayName": "Simon Willison",
  # ...
  </code></pre></div>
  <p dir="auto">If the resource you are fetching supports pagination you can use <code>--paginate key</code> to paginate through all of the rows in a specified key. For example, the following API has a <code>nextPageToken</code> key and a <code>files</code> list, suggesting it supports pagination:</p>
  <div class="snippet-clipboard-content notranslate position-relative overflow-auto" data-snippet-clipboard-copy-content="$ google-drive-to-sqlite get https://www.googleapis.com/drive/v3/files
  {
      "kind": "drive#fileList",
      "nextPageToken": "~!!~AI9...wogHHYlc=",
      "incompleteSearch": false,
      "files": [
          {
              "kind": "drive#file",
              "id": "1YEsITp_X8PtDUJWHGM0osT-TXAU1nr0e7RSWRM2Jpyg",
              "name": "Title of a spreadsheet",
              "mimeType": "application/vnd.google-apps.spreadsheet"
          },"><pre class="notranslate"><code>$ google-drive-to-sqlite get https://www.googleapis.com/drive/v3/files
  {
      "kind": "drive#fileList",
      "nextPageToken": "~!!~AI9...wogHHYlc=",
      "incompleteSearch": false,
      "files": [
          {
              "kind": "drive#file",
              "id": "1YEsITp_X8PtDUJWHGM0osT-TXAU1nr0e7RSWRM2Jpyg",
              "name": "Title of a spreadsheet",
              "mimeType": "application/vnd.google-apps.spreadsheet"
          },
  </code></pre></div>
  <p dir="auto">To paginate through everything in the <code>files</code> list you would use <code>--paginate files</code> like this:</p>
  <div class="snippet-clipboard-content notranslate position-relative overflow-auto" data-snippet-clipboard-copy-content="$ google-drive-to-sqlite get https://www.googleapis.com/drive/v3/files --paginate files
  [
    {
      "kind": "drive#file",
      "id": "1YEsITp_X8PtDUJWHGM0osT-TXAU1nr0e7RSWRM2Jpyg",
      "name": "Title of a spreadsheet",
      "mimeType": "application/vnd.google-apps.spreadsheet"
    },
    # ..."><pre class="notranslate"><code>$ google-drive-to-sqlite get https://www.googleapis.com/drive/v3/files --paginate files
  [
    {
      "kind": "drive#file",
      "id": "1YEsITp_X8PtDUJWHGM0osT-TXAU1nr0e7RSWRM2Jpyg",
      "name": "Title of a spreadsheet",
      "mimeType": "application/vnd.google-apps.spreadsheet"
    },
    # ...
  </code></pre></div>
  <p dir="auto">Add <code>--nl</code> to stream paginated data as newline-delimited JSON:</p>
  <div class="snippet-clipboard-content notranslate position-relative overflow-auto" data-snippet-clipboard-copy-content="$ google-drive-to-sqlite get https://www.googleapis.com/drive/v3/files --paginate files --nl
  {"kind": "drive#file", "id": "1YEsITp_X8PtDUJWHGM0osT-TXAU1nr0e7RSWRM2Jpyg", "name": "Title of a spreadsheet", "mimeType": "application/vnd.google-apps.spreadsheet"}
  {"kind": "drive#file", "id": "1E6Zg2X2bjjtPzVfX8YqdXZDCoB3AVA7i", "name": "Subfolder", "mimeType": "application/vnd.google-apps.folder"}"><pre class="notranslate"><code>$ google-drive-to-sqlite get https://www.googleapis.com/drive/v3/files --paginate files --nl
  {"kind": "drive#file", "id": "1YEsITp_X8PtDUJWHGM0osT-TXAU1nr0e7RSWRM2Jpyg", "name": "Title of a spreadsheet", "mimeType": "application/vnd.google-apps.spreadsheet"}
  {"kind": "drive#file", "id": "1E6Zg2X2bjjtPzVfX8YqdXZDCoB3AVA7i", "name": "Subfolder", "mimeType": "application/vnd.google-apps.folder"}
  </code></pre></div>
  <p dir="auto">Add <code>--stop-after 5</code> to stop after 5 records - useful for testing.</p>
  <p dir="auto">Full <code>--help</code>:</p>

  <div class="snippet-clipboard-content notranslate position-relative overflow-auto" data-snippet-clipboard-copy-content="Usage: google-drive-to-sqlite get [OPTIONS] URL

    Make an authenticated HTTP GET to the specified URL

  Options:
    -a, --auth FILE       Path to auth.json token file
    --paginate TEXT       Paginate through all results in this key
    --nl                  Output paginated data as newline-delimited JSON
    --stop-after INTEGER  Stop paginating after X results
    -v, --verbose         Send verbose output to stderr
    --help                Show this message and exit.
  "><pre class="notranslate"><code>Usage: google-drive-to-sqlite get [OPTIONS] URL

    Make an authenticated HTTP GET to the specified URL

  Options:
    -a, --auth FILE       Path to auth.json token file
    --paginate TEXT       Paginate through all results in this key
    --nl                  Output paginated data as newline-delimited JSON
    --stop-after INTEGER  Stop paginating after X results
    -v, --verbose         Send verbose output to stderr
    --help                Show this message and exit.

  </code></pre></div>

  <div class="markdown-heading" dir="auto"><h2 class="heading-element" dir="auto">Database schema</h2><a id="user-content-database-schema" class="anchor" aria-label="Permalink: Database schema" href="#database-schema"><svg class="octicon octicon-link" viewbox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
  <p dir="auto">The database created by this tool has the following schema:</p>

  <div class="highlight highlight-source-sql notranslate position-relative overflow-auto" dir="auto" data-snippet-clipboard-copy-content="CREATE TABLE [drive_users] (
     [permissionId] TEXT PRIMARY KEY,
     [kind] TEXT,
     [displayName] TEXT,
     [photoLink] TEXT,
     [me] INTEGER,
     [emailAddress] TEXT
  );
  CREATE TABLE [drive_folders] (
     [id] TEXT PRIMARY KEY,
     [_parent] TEXT,
     [_owner] TEXT,
     [lastModifyingUser] TEXT,
     [kind] TEXT,
     [name] TEXT,
     [mimeType] TEXT,
     [starred] INTEGER,
     [trashed] INTEGER,
     [explicitlyTrashed] INTEGER,
     [parents] TEXT,
     [spaces] TEXT,
     [version] TEXT,
     [webViewLink] TEXT,
     [iconLink] TEXT,
     [hasThumbnail] INTEGER,
     [thumbnailVersion] TEXT,
     [viewedByMe] INTEGER,
     [createdTime] TEXT,
     [modifiedTime] TEXT,
     [modifiedByMe] INTEGER,
     [shared] INTEGER,
     [ownedByMe] INTEGER,
     [viewersCanCopyContent] INTEGER,
     [copyRequiresWriterPermission] INTEGER,
     [writersCanShare] INTEGER,
     [folderColorRgb] TEXT,
     [quotaBytesUsed] TEXT,
     [isAppAuthorized] INTEGER,
     [linkShareMetadata] TEXT,
     FOREIGN KEY([_parent]) REFERENCES [drive_folders]([id]),
     FOREIGN KEY([_owner]) REFERENCES [drive_users]([permissionId]),
     FOREIGN KEY([lastModifyingUser]) REFERENCES [drive_users]([permissionId])
  );
  CREATE TABLE [drive_files] (
     [id] TEXT PRIMARY KEY,
     [_parent] TEXT,
     [_owner] TEXT,
     [lastModifyingUser] TEXT,
     [kind] TEXT,
     [name] TEXT,
     [mimeType] TEXT,
     [starred] INTEGER,
     [trashed] INTEGER,
     [explicitlyTrashed] INTEGER,
     [parents] TEXT,
     [spaces] TEXT,
     [version] TEXT,
     [webViewLink] TEXT,
     [iconLink] TEXT,
     [hasThumbnail] INTEGER,
     [thumbnailVersion] TEXT,
     [viewedByMe] INTEGER,
     [createdTime] TEXT,
     [modifiedTime] TEXT,
     [modifiedByMe] INTEGER,
     [shared] INTEGER,
     [ownedByMe] INTEGER,
     [viewersCanCopyContent] INTEGER,
     [copyRequiresWriterPermission] INTEGER,
     [writersCanShare] INTEGER,
     [quotaBytesUsed] TEXT,
     [isAppAuthorized] INTEGER,
     [linkShareMetadata] TEXT,
     FOREIGN KEY([_parent]) REFERENCES [drive_folders]([id]),
     FOREIGN KEY([_owner]) REFERENCES [drive_users]([permissionId]),
     FOREIGN KEY([lastModifyingUser]) REFERENCES [drive_users]([permissionId])
  );"><pre>CREATE TABLE [drive_users] (
     [permissionId] <span class="pl-k">TEXT</span> <span class="pl-k">PRIMARY KEY</span>,
     [kind] <span class="pl-k">TEXT</span>,
     [displayName] <span class="pl-k">TEXT</span>,
     [photoLink] <span class="pl-k">TEXT</span>,
     [me] <span class="pl-k">INTEGER</span>,
     [emailAddress] <span class="pl-k">TEXT</span>
  );
  CREATE TABLE [drive_folders] (
     [id] <span class="pl-k">TEXT</span> <span class="pl-k">PRIMARY KEY</span>,
     [_parent] <span class="pl-k">TEXT</span>,
     [_owner] <span class="pl-k">TEXT</span>,
     [lastModifyingUser] <span class="pl-k">TEXT</span>,
     [kind] <span class="pl-k">TEXT</span>,
     [name] <span class="pl-k">TEXT</span>,
     [mimeType] <span class="pl-k">TEXT</span>,
     [starred] <span class="pl-k">INTEGER</span>,
     [trashed] <span class="pl-k">INTEGER</span>,
     [explicitlyTrashed] <span class="pl-k">INTEGER</span>,
     [parents] <span class="pl-k">TEXT</span>,
     [spaces] <span class="pl-k">TEXT</span>,
     [version] <span class="pl-k">TEXT</span>,
     [webViewLink] <span class="pl-k">TEXT</span>,
     [iconLink] <span class="pl-k">TEXT</span>,
     [hasThumbnail] <span class="pl-k">INTEGER</span>,
     [thumbnailVersion] <span class="pl-k">TEXT</span>,
     [viewedByMe] <span class="pl-k">INTEGER</span>,
     [createdTime] <span class="pl-k">TEXT</span>,
     [modifiedTime] <span class="pl-k">TEXT</span>,
     [modifiedByMe] <span class="pl-k">INTEGER</span>,
     [shared] <span class="pl-k">INTEGER</span>,
     [ownedByMe] <span class="pl-k">INTEGER</span>,
     [viewersCanCopyContent] <span class="pl-k">INTEGER</span>,
     [copyRequiresWriterPermission] <span class="pl-k">INTEGER</span>,
     [writersCanShare] <span class="pl-k">INTEGER</span>,
     [folderColorRgb] <span class="pl-k">TEXT</span>,
     [quotaBytesUsed] <span class="pl-k">TEXT</span>,
     [isAppAuthorized] <span class="pl-k">INTEGER</span>,
     [linkShareMetadata] <span class="pl-k">TEXT</span>,
     <span class="pl-k">FOREIGN KEY</span>([_parent]) <span class="pl-k">REFERENCES</span> [drive_folders]([id]),
     <span class="pl-k">FOREIGN KEY</span>([_owner]) <span class="pl-k">REFERENCES</span> [drive_users]([permissionId]),
     <span class="pl-k">FOREIGN KEY</span>([lastModifyingUser]) <span class="pl-k">REFERENCES</span> [drive_users]([permissionId])
  );
  CREATE TABLE [drive_files] (
     [id] <span class="pl-k">TEXT</span> <span class="pl-k">PRIMARY KEY</span>,
     [_parent] <span class="pl-k">TEXT</span>,
     [_owner] <span class="pl-k">TEXT</span>,
     [lastModifyingUser] <span class="pl-k">TEXT</span>,
     [kind] <span class="pl-k">TEXT</span>,
     [name] <span class="pl-k">TEXT</span>,
     [mimeType] <span class="pl-k">TEXT</span>,
     [starred] <span class="pl-k">INTEGER</span>,
     [trashed] <span class="pl-k">INTEGER</span>,
     [explicitlyTrashed] <span class="pl-k">INTEGER</span>,
     [parents] <span class="pl-k">TEXT</span>,
     [spaces] <span class="pl-k">TEXT</span>,
     [version] <span class="pl-k">TEXT</span>,
     [webViewLink] <span class="pl-k">TEXT</span>,
     [iconLink] <span class="pl-k">TEXT</span>,
     [hasThumbnail] <span class="pl-k">INTEGER</span>,
     [thumbnailVersion] <span class="pl-k">TEXT</span>,
     [viewedByMe] <span class="pl-k">INTEGER</span>,
     [createdTime] <span class="pl-k">TEXT</span>,
     [modifiedTime] <span class="pl-k">TEXT</span>,
     [modifiedByMe] <span class="pl-k">INTEGER</span>,
     [shared] <span class="pl-k">INTEGER</span>,
     [ownedByMe] <span class="pl-k">INTEGER</span>,
     [viewersCanCopyContent] <span class="pl-k">INTEGER</span>,
     [copyRequiresWriterPermission] <span class="pl-k">INTEGER</span>,
     [writersCanShare] <span class="pl-k">INTEGER</span>,
     [quotaBytesUsed] <span class="pl-k">TEXT</span>,
     [isAppAuthorized] <span class="pl-k">INTEGER</span>,
     [linkShareMetadata] <span class="pl-k">TEXT</span>,
     <span class="pl-k">FOREIGN KEY</span>([_parent]) <span class="pl-k">REFERENCES</span> [drive_folders]([id]),
     <span class="pl-k">FOREIGN KEY</span>([_owner]) <span class="pl-k">REFERENCES</span> [drive_users]([permissionId]),
     <span class="pl-k">FOREIGN KEY</span>([lastModifyingUser]) <span class="pl-k">REFERENCES</span> [drive_users]([permissionId])
  );</pre></div>

  <div class="markdown-heading" dir="auto"><h2 class="heading-element" dir="auto">Thumbnails</h2><a id="user-content-thumbnails" class="anchor" aria-label="Permalink: Thumbnails" href="#thumbnails"><svg class="octicon octicon-link" viewbox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
  <p dir="auto">You can construct a thumbnail image for a known file ID using the following URL:</p>
  <div class="snippet-clipboard-content notranslate position-relative overflow-auto" data-snippet-clipboard-copy-content="https://drive.google.com/thumbnail?sz=w800-h800&id=FILE_ID"><pre class="notranslate"><code>https://drive.google.com/thumbnail?sz=w800-h800&id=FILE_ID
  </code></pre></div>
  <p dir="auto">Users who are signed into Google Drive and have permission to view a file will be redirected to a thumbnail version of that file. You can tweak the <code>w800</code> and <code>h800</code> parameters to request different thumbnail sizes.</p>
  <div class="markdown-heading" dir="auto"><h2 class="heading-element" dir="auto">Privacy policy</h2><a id="user-content-privacy-policy" class="anchor" aria-label="Permalink: Privacy policy" href="#privacy-policy"><svg class="octicon octicon-link" viewbox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
  <p dir="auto">This tool requests access to your Google Drive account in order to retrieve metadata about your files there. It also offers a feature that can download the content of those files.</p>
  <p dir="auto">The credentials used to access your account are stored in the <code>auth.json</code> file on your computer. The metadata and content retrieved from Google Drive is also stored only on your own personal computer.</p>
  <p dir="auto">At no point do the developers of this tool gain access to any of your data.</p>
  <div class="markdown-heading" dir="auto"><h2 class="heading-element" dir="auto">Development</h2><a id="user-content-development" class="anchor" aria-label="Permalink: Development" href="#development"><svg class="octicon octicon-link" viewbox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
  <p dir="auto">To contribute to this tool, first checkout the code. Then create a new virtual environment:</p>
  <div class="snippet-clipboard-content notranslate position-relative overflow-auto" data-snippet-clipboard-copy-content="cd google-drive-to-sqlite
  python -m venv venv
  source venv/bin/activate"><pre class="notranslate"><code>cd google-drive-to-sqlite
  python -m venv venv
  source venv/bin/activate
  </code></pre></div>
  <p dir="auto">Or if you are using <code>pipenv</code>:</p>
  <div class="snippet-clipboard-content notranslate position-relative overflow-auto" data-snippet-clipboard-copy-content="pipenv shell"><pre class="notranslate"><code>pipenv shell
  </code></pre></div>
  <p dir="auto">Now install the dependencies and test dependencies:</p>
  <div class="snippet-clipboard-content notranslate position-relative overflow-auto" data-snippet-clipboard-copy-content="pip install -e '.[test]'"><pre class="notranslate"><code>pip install -e '.[test]'
  </code></pre></div>
  <p dir="auto">To run the tests:</p>
  <div class="snippet-clipboard-content notranslate position-relative overflow-auto" data-snippet-clipboard-copy-content="pytest"><pre class="notranslate"><code>pytest
  </code></pre></div>
  </article></div>


node_id: R_kgDOG383_g
name: sqlite-colorbrewer
full_name: eyeseast/sqlite-colorbrewer
html_url: https://github.com/eyeseast/sqlite-colorbrewer
description: A custom function to use ColorBrewer scales in SQLite queries
created_at: 2022-02-19T21:53:46Z
updated_at: 2023-09-01T18:20:43Z
pushed_at: 2022-03-02T03:04:56Z
language: Python
license: apache-2.0
topics: []
visibility: public
default_branch: main
permissions: {"admin": false, "maintain": false, "push": false, "triage": false, "pull": true}
readme:
  # sqlite-colorbrewer

  [![PyPI](https://img.shields.io/pypi/v/sqlite-colorbrewer.svg)](https://pypi.org/project/sqlite-colorbrewer/)
  [![Changelog](https://img.shields.io/github/v/release/eyeseast/sqlite-colorbrewer?include_prereleases&label=changelog)](https://github.com/eyeseast/sqlite-colorbrewer/releases)
  [![Tests](https://github.com/eyeseast/sqlite-colorbrewer/workflows/Test/badge.svg)](https://github.com/eyeseast/sqlite-colorbrewer/actions?query=workflow%3ATest)
  [![License](https://img.shields.io/badge/license-Apache%202.0-blue.svg)](https://github.com/eyeseast/sqlite-colorbrewer/blob/main/LICENSE)

  A custom function to use [ColorBrewer](https://colorbrewer2.org/) scales in SQLite queries.

  Colors are exported from [here](https://colorbrewer2.org/export/colorbrewer.json).

  ## Installation

  To install as a Python library and use with the [standard SQLite3 module](https://docs.python.org/3/library/sqlite3.html):

      pip install sqlite-colorbrewer

  To install this plugin in the same environment as Datasette.

      datasette install sqlite-colorbrewer

  ## Usage

  If you're using this library with Datasette, it will be automatically registered as a plugin and available for use in SQL queries, like so:

  ```sql
  SELECT colorbrewer('Blues', 9, 0);
  ```

  That will return a single value: `"rgb(247,251,255)"`

  To use with a SQLite connection outside of Datasette, use the `register` function:

  ```python
  >>> import sqlite3
  >>> import sqlite_colorbrewer

  >>> conn = sqlite3.connect(':memory')
  >>> sqlite_colorbrewer.register(conn)

  >>> cursor = conn.execute("SELECT colorbrewer('Blues', 9, 0);")
  >>> result = next(cursor)
  >>> print(result)
  rgb(247,251,255)
  ```

  ## Development

  To set up this plugin locally, first checkout the code. Then create a new virtual environment:

      cd sqlite-colorbrewer
      python3 -mvenv venv
      source venv/bin/activate

  Or if you are using `pipenv`:

      pipenv shell

  Now install the dependencies and test dependencies:

      pip install -e '.[test]'

  To run the tests:

      pytest

  To build `sqlite_colorbrewer/colorbrewer.py`:

      ./json_to_python.py
      black . # to format the resulting file

  ## ColorBrewer

  Copyright (c) 2002 Cynthia Brewer, Mark Harrower, and The Pennsylvania State University.

  Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at

  http://www.apache.org/licenses/LICENSE-2.0

  Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.

  See the [ColorBrewer updates](http://www.personal.psu.edu/cab38/ColorBrewer/ColorBrewer_updates.html) for updates to copyright information.

readme_html:
  <div id="readme" class="md" data-path="README.md"><article class="markdown-body entry-content container-lg" itemprop="text"><div class="markdown-heading" dir="auto"><h1 class="heading-element" dir="auto">sqlite-colorbrewer</h1><a id="user-content-sqlite-colorbrewer" class="anchor" aria-label="Permalink: sqlite-colorbrewer" href="#sqlite-colorbrewer"><svg class="octicon octicon-link" viewbox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
  <p dir="auto"><a href="https://pypi.org/project/sqlite-colorbrewer/" rel="nofollow"><img src="https://img.shields.io/pypi/v/sqlite-colorbrewer.svg" alt="PyPI" style="max-width: 100%;" data-camo-src="https://camo.githubusercontent.com/75180155998cf9ef1e36644c39a42288e3c57d9dccdf04d4784a657e39a21250/68747470733a2f2f696d672e736869656c64732e696f2f707970692f762f73716c6974652d636f6c6f726272657765722e737667"></a>
  <a href="https://github.com/eyeseast/sqlite-colorbrewer/releases"><img src="https://img.shields.io/github/v/release/eyeseast/sqlite-colorbrewer?include_prereleases&label=changelog" alt="Changelog" style="max-width: 100%;" data-camo-src="https://camo.githubusercontent.com/1891e841f0fc1225ec184ca92abee2a1baa11651b7eff841beff359036fbcdd2/68747470733a2f2f696d672e736869656c64732e696f2f6769746875622f762f72656c656173652f65796573656173742f73716c6974652d636f6c6f726272657765723f696e636c7564655f70726572656c6561736573266c6162656c3d6368616e67656c6f67"></a>
  <a href="https://github.com/eyeseast/sqlite-colorbrewer/actions?query=workflow%3ATest"><img src="https://github.com/eyeseast/sqlite-colorbrewer/workflows/Test/badge.svg" alt="Tests" style="max-width: 100%;"></a>
  <a href="https://github.com/eyeseast/sqlite-colorbrewer/blob/main/LICENSE"><img src="https://img.shields.io/badge/license-Apache%202.0-blue.svg" alt="License" style="max-width: 100%;" data-camo-src="https://camo.githubusercontent.com/c355f200ea90fddaa407b6eaab303663a669248ea3ca7b1fcf77dbe04ff5f48c/68747470733a2f2f696d672e736869656c64732e696f2f62616467652f6c6963656e73652d417061636865253230322e302d626c75652e737667"></a></p>
  <p dir="auto">A custom function to use <a href="https://colorbrewer2.org/" rel="nofollow">ColorBrewer</a> scales in SQLite queries.</p>
  <p dir="auto">Colors are exported from <a href="https://colorbrewer2.org/export/colorbrewer.json" rel="nofollow">here</a>.</p>
  <div class="markdown-heading" dir="auto"><h2 class="heading-element" dir="auto">Installation</h2><a id="user-content-installation" class="anchor" aria-label="Permalink: Installation" href="#installation"><svg class="octicon octicon-link" viewbox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
  <p dir="auto">To install as a Python library and use with the <a href="https://docs.python.org/3/library/sqlite3.html" rel="nofollow">standard SQLite3 module</a>:</p>
  <div class="snippet-clipboard-content notranslate position-relative overflow-auto" data-snippet-clipboard-copy-content="pip install sqlite-colorbrewer"><pre class="notranslate"><code>pip install sqlite-colorbrewer
  </code></pre></div>
  <p dir="auto">To install this plugin in the same environment as Datasette.</p>
  <div class="snippet-clipboard-content notranslate position-relative overflow-auto" data-snippet-clipboard-copy-content="datasette install sqlite-colorbrewer"><pre class="notranslate"><code>datasette install sqlite-colorbrewer
  </code></pre></div>
  <div class="markdown-heading" dir="auto"><h2 class="heading-element" dir="auto">Usage</h2><a id="user-content-usage" class="anchor" aria-label="Permalink: Usage" href="#usage"><svg class="octicon octicon-link" viewbox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
  <p dir="auto">If you're using this library with Datasette, it will be automatically registered as a plugin and available for use in SQL queries, like so:</p>
  <div class="highlight highlight-source-sql notranslate position-relative overflow-auto" dir="auto" data-snippet-clipboard-copy-content="SELECT colorbrewer('Blues', 9, 0);"><pre><span class="pl-k">SELECT</span> colorbrewer(<span class="pl-s"><span class="pl-pds">'</span>Blues<span class="pl-pds">'</span></span>, <span class="pl-c1">9</span>, <span class="pl-c1">0</span>);</pre></div>
  <p dir="auto">That will return a single value: <code>"rgb(247,251,255)"</code></p>
  <p dir="auto">To use with a SQLite connection outside of Datasette, use the <code>register</code> function:</p>
  <div class="highlight highlight-source-python notranslate position-relative overflow-auto" dir="auto" data-snippet-clipboard-copy-content=">>> import sqlite3
  >>> import sqlite_colorbrewer

  >>> conn = sqlite3.connect(':memory')
  >>> sqlite_colorbrewer.register(conn)

  >>> cursor = conn.execute("SELECT colorbrewer('Blues', 9, 0);")
  >>> result = next(cursor)
  >>> print(result)
  rgb(247,251,255)"><pre><span class="pl-c1">></span><span class="pl-c1">>></span> <span class="pl-k">import</span> <span class="pl-s1">sqlite3</span>
  <span class="pl-c1">></span><span class="pl-c1">>></span> <span class="pl-k">import</span> <span class="pl-s1">sqlite_colorbrewer</span>

  <span class="pl-c1">></span><span class="pl-c1">>></span> <span class="pl-s1">conn</span> <span class="pl-c1">=</span> <span class="pl-s1">sqlite3</span>.<span class="pl-c1">connect</span>(<span class="pl-s">':memory'</span>)
  <span class="pl-c1">></span><span class="pl-c1">>></span> <span class="pl-s1">sqlite_colorbrewer</span>.<span class="pl-c1">register</span>(<span class="pl-s1">conn</span>)

  <span class="pl-c1">></span><span class="pl-c1">>></span> <span class="pl-s1">cursor</span> <span class="pl-c1">=</span> <span class="pl-s1">conn</span>.<span class="pl-c1">execute</span>(<span class="pl-s">"SELECT colorbrewer('Blues', 9, 0);"</span>)
  <span class="pl-c1">></span><span class="pl-c1">>></span> <span class="pl-s1">result</span> <span class="pl-c1">=</span> <span class="pl-en">next</span>(<span class="pl-s1">cursor</span>)
  <span class="pl-c1">></span><span class="pl-c1">>></span> <span class="pl-en">print</span>(<span class="pl-s1">result</span>)
  <span class="pl-en">rgb</span>(<span class="pl-c1">247</span>,<span class="pl-c1">251</span>,<span class="pl-c1">255</span>)</pre></div>
  <div class="markdown-heading" dir="auto"><h2 class="heading-element" dir="auto">Development</h2><a id="user-content-development" class="anchor" aria-label="Permalink: Development" href="#development"><svg class="octicon octicon-link" viewbox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
  <p dir="auto">To set up this plugin locally, first checkout the code. Then create a new virtual environment:</p>
  <div class="snippet-clipboard-content notranslate position-relative overflow-auto" data-snippet-clipboard-copy-content="cd sqlite-colorbrewer
  python3 -mvenv venv
  source venv/bin/activate"><pre class="notranslate"><code>cd sqlite-colorbrewer
  python3 -mvenv venv
  source venv/bin/activate
  </code></pre></div>
  <p dir="auto">Or if you are using <code>pipenv</code>:</p>
  <div class="snippet-clipboard-content notranslate position-relative overflow-auto" data-snippet-clipboard-copy-content="pipenv shell"><pre class="notranslate"><code>pipenv shell
  </code></pre></div>
  <p dir="auto">Now install the dependencies and test dependencies:</p>
  <div class="snippet-clipboard-content notranslate position-relative overflow-auto" data-snippet-clipboard-copy-content="pip install -e '.[test]'"><pre class="notranslate"><code>pip install -e '.[test]'
  </code></pre></div>
  <p dir="auto">To run the tests:</p>
  <div class="snippet-clipboard-content notranslate position-relative overflow-auto" data-snippet-clipboard-copy-content="pytest"><pre class="notranslate"><code>pytest
  </code></pre></div>
  <p dir="auto">To build <code>sqlite_colorbrewer/colorbrewer.py</code>:</p>
  <div class="snippet-clipboard-content notranslate position-relative overflow-auto" data-snippet-clipboard-copy-content="./json_to_python.py
  black . # to format the resulting file"><pre class="notranslate"><code>./json_to_python.py
  black . # to format the resulting file
  </code></pre></div>
  <div class="markdown-heading" dir="auto"><h2 class="heading-element" dir="auto">ColorBrewer</h2><a id="user-content-colorbrewer" class="anchor" aria-label="Permalink: ColorBrewer" href="#colorbrewer"><svg class="octicon octicon-link" viewbox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
  <p dir="auto">Copyright (c) 2002 Cynthia Brewer, Mark Harrower, and The Pennsylvania State University.</p>
  <p dir="auto">Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. You may obtain a copy of the License at</p>
  <p dir="auto"><a href="http://www.apache.org/licenses/LICENSE-2.0" rel="nofollow">http://www.apache.org/licenses/LICENSE-2.0</a></p>
  <p dir="auto">Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions and limitations under the License.</p>
  <p dir="auto">See the <a href="http://www.personal.psu.edu/cab38/ColorBrewer/ColorBrewer_updates.html" rel="nofollow">ColorBrewer updates</a> for updates to copyright information.</p>
  </article></div>


node_id: R_kgDOG5dZxg
name: datasette-redirect-forbidden
full_name: simonw/datasette-redirect-forbidden
html_url: https://github.com/simonw/datasette-redirect-forbidden
description: Redirect forbidden requests to a login page
created_at: 2022-02-23T20:59:26Z
updated_at: 2023-04-14T23:50:54Z
pushed_at: 2022-02-23T22:02:38Z
language: Python
license: apache-2.0
topics: []
visibility: public
default_branch: main
permissions: {"admin": false, "maintain": false, "push": false, "triage": false, "pull": true}
template_repository:
[cell repeats an earlier one in this file: 6377 characters]

readme:
  # datasette-redirect-forbidden

  [![PyPI](https://img.shields.io/pypi/v/datasette-redirect-forbidden.svg)](https://pypi.org/project/datasette-redirect-forbidden/)
  [![Changelog](https://img.shields.io/github/v/release/simonw/datasette-redirect-forbidden?include_prereleases&label=changelog)](https://github.com/simonw/datasette-redirect-forbidden/releases)
  [![Tests](https://github.com/simonw/datasette-redirect-forbidden/workflows/Test/badge.svg)](https://github.com/simonw/datasette-redirect-forbidden/actions?query=workflow%3ATest)
  [![License](https://img.shields.io/badge/license-Apache%202.0-blue.svg)](https://github.com/simonw/datasette-redirect-forbidden/blob/main/LICENSE)

  Redirect forbidden requests to a login page

  ## Installation

  Install this plugin in the same environment as Datasette.

      $ datasette install datasette-redirect-forbidden

  ## Usage

  Add the following to your `metadata.yml` (or `metadata.json`) file to configure the plugin:

  ```yaml
  plugins:
    datasette-redirect-forbidden:
      redirect_to: /-/login
  ```
  Any 403 forbidden pages will redirect to the specified page.

  ## Development

  To set up this plugin locally, first checkout the code. Then create a new virtual environment:

      cd datasette-redirect-forbidden
      python3 -mvenv venv
      source venv/bin/activate

  Or if you are using `pipenv`:

      pipenv shell

  Now install the dependencies and test dependencies:

      pip install -e '.[test]'

  To run the tests:

      pytest

readme_html:
  <div id="readme" class="md" data-path="README.md"><article class="markdown-body entry-content container-lg" itemprop="text"><div class="markdown-heading" dir="auto"><h1 class="heading-element" dir="auto">datasette-redirect-forbidden</h1><a id="user-content-datasette-redirect-forbidden" class="anchor" aria-label="Permalink: datasette-redirect-forbidden" href="#datasette-redirect-forbidden"><svg class="octicon octicon-link" viewbox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
  <p dir="auto"><a href="https://pypi.org/project/datasette-redirect-forbidden/" rel="nofollow"><img src="https://img.shields.io/pypi/v/datasette-redirect-forbidden.svg" alt="PyPI" style="max-width: 100%;" data-camo-src="https://camo.githubusercontent.com/798f0162b6e762ce80d0623a85ca8f37d30815deaa884fa0bd21302727753802/68747470733a2f2f696d672e736869656c64732e696f2f707970692f762f6461746173657474652d72656469726563742d666f7262696464656e2e737667"></a>
  <a href="https://github.com/simonw/datasette-redirect-forbidden/releases"><img src="https://img.shields.io/github/v/release/simonw/datasette-redirect-forbidden?include_prereleases&label=changelog" alt="Changelog" style="max-width: 100%;" data-camo-src="https://camo.githubusercontent.com/c30f936efca5df7875ad585ec4ebd2418a9c7ad038ec3c9ea644992e3ba14e41/68747470733a2f2f696d672e736869656c64732e696f2f6769746875622f762f72656c656173652f73696d6f6e772f6461746173657474652d72656469726563742d666f7262696464656e3f696e636c7564655f70726572656c6561736573266c6162656c3d6368616e67656c6f67"></a>
  <a href="https://github.com/simonw/datasette-redirect-forbidden/actions?query=workflow%3ATest"><img src="https://github.com/simonw/datasette-redirect-forbidden/workflows/Test/badge.svg" alt="Tests" style="max-width: 100%;"></a>
  <a href="https://github.com/simonw/datasette-redirect-forbidden/blob/main/LICENSE"><img src="https://img.shields.io/badge/license-Apache%202.0-blue.svg" alt="License" style="max-width: 100%;" data-camo-src="https://camo.githubusercontent.com/c355f200ea90fddaa407b6eaab303663a669248ea3ca7b1fcf77dbe04ff5f48c/68747470733a2f2f696d672e736869656c64732e696f2f62616467652f6c6963656e73652d417061636865253230322e302d626c75652e737667"></a></p>
  <p dir="auto">Redirect forbidden requests to a login page</p>
  <div class="markdown-heading" dir="auto"><h2 class="heading-element" dir="auto">Installation</h2><a id="user-content-installation" class="anchor" aria-label="Permalink: Installation" href="#installation"><svg class="octicon octicon-link" viewbox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
  <p dir="auto">Install this plugin in the same environment as Datasette.</p>
  <div class="snippet-clipboard-content notranslate position-relative overflow-auto" data-snippet-clipboard-copy-content="$ datasette install datasette-redirect-forbidden"><pre class="notranslate"><code>$ datasette install datasette-redirect-forbidden
  </code></pre></div>
  <div class="markdown-heading" dir="auto"><h2 class="heading-element" dir="auto">Usage</h2><a id="user-content-usage" class="anchor" aria-label="Permalink: Usage" href="#usage"><svg class="octicon octicon-link" viewbox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
  <p dir="auto">Add the following to your <code>metadata.yml</code> (or <code>metadata.json</code>) file to configure the plugin:</p>
  <div class="highlight highlight-source-yaml notranslate position-relative overflow-auto" dir="auto" data-snippet-clipboard-copy-content="plugins:
    datasette-redirect-forbidden:
      redirect_to: /-/login"><pre><span class="pl-ent">plugins</span>:
    <span class="pl-ent">datasette-redirect-forbidden</span>:
      <span class="pl-ent">redirect_to</span>: <span class="pl-s">/-/login</span></pre></div>
  <p dir="auto">Any 403 forbidden pages will redirect to the specified page.</p>
  <div class="markdown-heading" dir="auto"><h2 class="heading-element" dir="auto">Development</h2><a id="user-content-development" class="anchor" aria-label="Permalink: Development" href="#development"><svg class="octicon octicon-link" viewbox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
  <p dir="auto">To set up this plugin locally, first checkout the code. Then create a new virtual environment:</p>
  <div class="snippet-clipboard-content notranslate position-relative overflow-auto" data-snippet-clipboard-copy-content="cd datasette-redirect-forbidden
  python3 -mvenv venv
  source venv/bin/activate"><pre class="notranslate"><code>cd datasette-redirect-forbidden
  python3 -mvenv venv
  source venv/bin/activate
  </code></pre></div>
  <p dir="auto">Or if you are using <code>pipenv</code>:</p>
  <div class="snippet-clipboard-content notranslate position-relative overflow-auto" data-snippet-clipboard-copy-content="pipenv shell"><pre class="notranslate"><code>pipenv shell
  </code></pre></div>
  <p dir="auto">Now install the dependencies and test dependencies:</p>
  <div class="snippet-clipboard-content notranslate position-relative overflow-auto" data-snippet-clipboard-copy-content="pip install -e '.[test]'"><pre class="notranslate"><code>pip install -e '.[test]'
  </code></pre></div>
  <p dir="auto">To run the tests:</p>
  <div class="snippet-clipboard-content notranslate position-relative overflow-auto" data-snippet-clipboard-copy-content="pytest"><pre class="notranslate"><code>pytest
  </code></pre></div>
  </article></div>


node_id: R_kgDOG-A5Ww
name: shot-scraper
full_name: simonw/shot-scraper
html_url: https://github.com/simonw/shot-scraper
description: A command-line utility for taking automated screenshots of websites
created_at: 2022-03-08T21:21:02Z
updated_at: 2026-04-08T16:41:56Z
pushed_at: 2026-02-01T19:02:50Z
homepage: https://shot-scraper.datasette.io
language: Python
license: apache-2.0
topics: ["playwright", "playwright-python", "scraping", "screenshot-utility", "screenshots"]
visibility: public
default_branch: main
permissions: {"admin": false, "maintain": false, "push": false, "triage": false, "pull": false}
pull_request_creation_policy: all
readme:
  # shot-scraper

  [![PyPI](https://img.shields.io/pypi/v/shot-scraper.svg)](https://pypi.org/project/shot-scraper/)
  [![Changelog](https://img.shields.io/github/v/release/simonw/shot-scraper?include_prereleases&label=changelog)](https://github.com/simonw/shot-scraper/releases)
  [![Tests](https://github.com/simonw/shot-scraper/workflows/Test/badge.svg)](https://github.com/simonw/shot-scraper/actions?query=workflow%3ATest)
  [![License](https://img.shields.io/badge/license-Apache%202.0-blue.svg)](https://github.com/simonw/shot-scraper/blob/master/LICENSE)
  [![discord](https://img.shields.io/discord/823971286308356157?label=discord)](https://discord.gg/EE7Hx4Kbny)

  A command-line utility for taking automated screenshots of websites

  For background on this project see [shot-scraper: automated screenshots for documentation, built on Playwright](https://simonwillison.net/2022/Mar/10/shot-scraper/).

  ## Documentation

  - [Full documentation for shot-scraper](https://shot-scraper.datasette.io/)
  - [Tutorial: Automating screenshots for the Datasette documentation using shot-scraper](https://simonwillison.net/2022/Oct/14/automating-screenshots/)
  - [Release notes](https://github.com/simonw/shot-scraper/releases)

  ## Get started with GitHub Actions

  To get started without installing any software, use the [shot-scraper-template](https://github.com/simonw/shot-scraper-template) template to create your own GitHub repository which takes screenshots of a page using `shot-scraper`. See [Instantly create a GitHub repository to take screenshots of a web page](https://simonwillison.net/2022/Mar/14/shot-scraper-template/) for details.

  ## Quick installation

  You can install the `shot-scraper` CLI tool using [pip](https://pip.pypa.io/):

      pip install shot-scraper
      # Now install the browser it needs:
      shot-scraper install

  ## Taking your first screenshot

  You can take a screenshot of a web page like this:

      shot-scraper https://datasette.io/

  This will create a screenshot in a file called `datasette-io.png`.

  Many more options are available, see [Taking a screenshot](https://shot-scraper.datasette.io/en/stable/screenshots.html) for details.

  ## Examples

  - The [shot-scraper-demo](https://github.com/simonw/shot-scraper-demo) repository uses this tool to capture recently spotted owls in El Granada, CA according to [this page](https://www.owlsnearme.com/?place=127871), and to  generate an annotated screenshot illustrating a Datasette feature as described [in my blog](https://simonwillison.net/2022/Mar/10/shot-scraper/#a-complex-example).
  - The [Datasette Documentation](https://docs.datasette.io/en/latest/) uses screenshots taken by `shot-scraper` running in the [simonw/datasette-screenshots](https://github.com/simonw/datasette-screenshots) GitHub repository, described in detail in [Automating screenshots for the Datasette documentation using shot-scraper](https://simonwillison.net/2022/Oct/14/automating-screenshots/).
  - Ben Welsh built [@newshomepages](https://twitter.com/newshomepages), a Twitter bot that uses `shot-scraper` and GitHub Actions to take screenshots of news website homepages and publish them to Twitter. The code for that lives in [palewire/news-homepages](https://github.com/palewire/news-homepages).
  - [scrape-hacker-news-by-domain](https://github.com/simonw/scrape-hacker-news-by-domain) uses `shot-scraper javascript` to scrape a web page. See [Scraping web pages from the command-line with shot-scraper](https://simonwillison.net/2022/Mar/14/scraping-web-pages-shot-scraper/) for details of how this works.
  - Reuters uses shot-scraper to generate regularly updating data dashboards [for email newsletters](https://twitter.com/palewire/status/1658069533763026944).

readme_html:
  <div id="readme" class="md" data-path="README.md"><article class="markdown-body entry-content container-lg" itemprop="text"><div class="markdown-heading" dir="auto"><h1 class="heading-element" dir="auto">shot-scraper</h1><a id="user-content-shot-scraper" class="anchor" aria-label="Permalink: shot-scraper" href="#shot-scraper"><svg class="octicon octicon-link" viewbox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
  <p dir="auto"><a href="https://pypi.org/project/shot-scraper/" rel="nofollow"><img src="https://img.shields.io/pypi/v/shot-scraper.svg" alt="PyPI" style="max-width: 100%;" data-camo-src="https://camo.githubusercontent.com/d03992545374f5df21231c0b67397b47b7f180f7ff7c5748e0c6eb9d5859381a/68747470733a2f2f696d672e736869656c64732e696f2f707970692f762f73686f742d736372617065722e737667"></a>
  <a href="https://github.com/simonw/shot-scraper/releases"><img src="https://img.shields.io/github/v/release/simonw/shot-scraper?include_prereleases&label=changelog" alt="Changelog" style="max-width: 100%;" data-camo-src="https://camo.githubusercontent.com/c2df2c4205bb47abba3b47abf79c24ec486619a835feea812f806e3481f522c6/68747470733a2f2f696d672e736869656c64732e696f2f6769746875622f762f72656c656173652f73696d6f6e772f73686f742d736372617065723f696e636c7564655f70726572656c6561736573266c6162656c3d6368616e67656c6f67"></a>
  <a href="https://github.com/simonw/shot-scraper/actions?query=workflow%3ATest"><img src="https://github.com/simonw/shot-scraper/workflows/Test/badge.svg" alt="Tests" style="max-width: 100%;"></a>
  <a href="https://github.com/simonw/shot-scraper/blob/master/LICENSE"><img src="https://img.shields.io/badge/license-Apache%202.0-blue.svg" alt="License" style="max-width: 100%;" data-camo-src="https://camo.githubusercontent.com/c355f200ea90fddaa407b6eaab303663a669248ea3ca7b1fcf77dbe04ff5f48c/68747470733a2f2f696d672e736869656c64732e696f2f62616467652f6c6963656e73652d417061636865253230322e302d626c75652e737667"></a>
  <a href="https://discord.gg/EE7Hx4Kbny" rel="nofollow"><img src="https://img.shields.io/discord/823971286308356157?label=discord" alt="discord" style="max-width: 100%;" data-camo-src="https://camo.githubusercontent.com/94e396359356f1b1dd8fed242a2f694cb88a3b94a9eef104404225094357e912/68747470733a2f2f696d672e736869656c64732e696f2f646973636f72642f3832333937313238363330383335363135373f6c6162656c3d646973636f7264"></a></p>
  <p dir="auto">A command-line utility for taking automated screenshots of websites</p>
  <p dir="auto">For background on this project see <a href="https://simonwillison.net/2022/Mar/10/shot-scraper/" rel="nofollow">shot-scraper: automated screenshots for documentation, built on Playwright</a>.</p>
  <div class="markdown-heading" dir="auto"><h2 class="heading-element" dir="auto">Documentation</h2><a id="user-content-documentation" class="anchor" aria-label="Permalink: Documentation" href="#documentation"><svg class="octicon octicon-link" viewbox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
  <ul dir="auto">
  <li><a href="https://shot-scraper.datasette.io/" rel="nofollow">Full documentation for shot-scraper</a></li>
  <li><a href="https://simonwillison.net/2022/Oct/14/automating-screenshots/" rel="nofollow">Tutorial: Automating screenshots for the Datasette documentation using shot-scraper</a></li>
  <li><a href="https://github.com/simonw/shot-scraper/releases">Release notes</a></li>
  </ul>
  <div class="markdown-heading" dir="auto"><h2 class="heading-element" dir="auto">Get started with GitHub Actions</h2><a id="user-content-get-started-with-github-actions" class="anchor" aria-label="Permalink: Get started with GitHub Actions" href="#get-started-with-github-actions"><svg class="octicon octicon-link" viewbox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
  <p dir="auto">To get started without installing any software, use the <a href="https://github.com/simonw/shot-scraper-template">shot-scraper-template</a> template to create your own GitHub repository which takes screenshots of a page using <code>shot-scraper</code>. See <a href="https://simonwillison.net/2022/Mar/14/shot-scraper-template/" rel="nofollow">Instantly create a GitHub repository to take screenshots of a web page</a> for details.</p>
  <div class="markdown-heading" dir="auto"><h2 class="heading-element" dir="auto">Quick installation</h2><a id="user-content-quick-installation" class="anchor" aria-label="Permalink: Quick installation" href="#quick-installation"><svg class="octicon octicon-link" viewbox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
  <p dir="auto">You can install the <code>shot-scraper</code> CLI tool using <a href="https://pip.pypa.io/" rel="nofollow">pip</a>:</p>
  <div class="snippet-clipboard-content notranslate position-relative overflow-auto" data-snippet-clipboard-copy-content="pip install shot-scraper
  # Now install the browser it needs:
  shot-scraper install"><pre class="notranslate"><code>pip install shot-scraper
  # Now install the browser it needs:
  shot-scraper install
  </code></pre></div>
  <div class="markdown-heading" dir="auto"><h2 class="heading-element" dir="auto">Taking your first screenshot</h2><a id="user-content-taking-your-first-screenshot" class="anchor" aria-label="Permalink: Taking your first screenshot" href="#taking-your-first-screenshot"><svg class="octicon octicon-link" viewbox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
  <p dir="auto">You can take a screenshot of a web page like this:</p>
  <div class="snippet-clipboard-content notranslate position-relative overflow-auto" data-snippet-clipboard-copy-content="shot-scraper https://datasette.io/"><pre class="notranslate"><code>shot-scraper https://datasette.io/
  </code></pre></div>
  <p dir="auto">This will create a screenshot in a file called <code>datasette-io.png</code>.</p>
  <p dir="auto">Many more options are available, see <a href="https://shot-scraper.datasette.io/en/stable/screenshots.html" rel="nofollow">Taking a screenshot</a> for details.</p>
  <div class="markdown-heading" dir="auto"><h2 class="heading-element" dir="auto">Examples</h2><a id="user-content-examples" class="anchor" aria-label="Permalink: Examples" href="#examples"><svg class="octicon octicon-link" viewbox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
  <ul dir="auto">
  <li>The <a href="https://github.com/simonw/shot-scraper-demo">shot-scraper-demo</a> repository uses this tool to capture recently spotted owls in El Granada, CA according to <a href="https://www.owlsnearme.com/?place=127871" rel="nofollow">this page</a>, and to  generate an annotated screenshot illustrating a Datasette feature as described <a href="https://simonwillison.net/2022/Mar/10/shot-scraper/#a-complex-example" rel="nofollow">in my blog</a>.</li>
  <li>The <a href="https://docs.datasette.io/en/latest/" rel="nofollow">Datasette Documentation</a> uses screenshots taken by <code>shot-scraper</code> running in the <a href="https://github.com/simonw/datasette-screenshots">simonw/datasette-screenshots</a> GitHub repository, described in detail in <a href="https://simonwillison.net/2022/Oct/14/automating-screenshots/" rel="nofollow">Automating screenshots for the Datasette documentation using shot-scraper</a>.</li>
  <li>Ben Welsh built <a href="https://twitter.com/newshomepages" rel="nofollow">@newshomepages</a>, a Twitter bot that uses <code>shot-scraper</code> and GitHub Actions to take screenshots of news website homepages and publish them to Twitter. The code for that lives in <a href="https://github.com/palewire/news-homepages">palewire/news-homepages</a>.</li>
  <li><a href="https://github.com/simonw/scrape-hacker-news-by-domain">scrape-hacker-news-by-domain</a> uses <code>shot-scraper javascript</code> to scrape a web page. See <a href="https://simonwillison.net/2022/Mar/14/scraping-web-pages-shot-scraper/" rel="nofollow">Scraping web pages from the command-line with shot-scraper</a> for details of how this works.</li>
  <li>Reuters uses shot-scraper to generate regularly updating data dashboards <a href="https://twitter.com/palewire/status/1658069533763026944" rel="nofollow">for email newsletters</a>.</li>
  </ul>
  </article></div>


node_id: R_kgDOHAjKFQ
name: datasette-hashed-urls
full_name: simonw/datasette-hashed-urls
html_url: https://github.com/simonw/datasette-hashed-urls
description: Optimize Datasette performance behind a caching proxy
created_at: 2022-03-15T21:31:52Z
updated_at: 2025-11-02T20:10:45Z
pushed_at: 2025-11-02T20:10:42Z
language: Python
license: apache-2.0
topics: []
visibility: public
default_branch: main
permissions: {"admin": false, "maintain": false, "push": false, "triage": false, "pull": false}
readme:
  # datasette-hashed-urls

  [![PyPI](https://img.shields.io/pypi/v/datasette-hashed-urls.svg)](https://pypi.org/project/datasette-hashed-urls/)
  [![Changelog](https://img.shields.io/github/v/release/simonw/datasette-hashed-urls?include_prereleases&label=changelog)](https://github.com/simonw/datasette-hashed-urls/releases)
  [![Tests](https://github.com/simonw/datasette-hashed-urls/workflows/Test/badge.svg)](https://github.com/simonw/datasette-hashed-urls/actions?query=workflow%3ATest)
  [![License](https://img.shields.io/badge/license-Apache%202.0-blue.svg)](https://github.com/simonw/datasette-hashed-urls/blob/main/LICENSE)

  Optimize Datasette performance behind a caching proxy

  When you open a database file in immutable mode using the `-i` option, Datasette calculates a SHA-256 hash of the contents of that file on startup.

  This content hash can then optionally be used to create URLs that are guaranteed to change if the contents of the file changes in the future.

  The result is pages  that can be cached indefinitely by both browsers and caching proxies - providing a significant performance boost.

  ## Demo

  A demo of this plugin is running at https://datasette-hashed-urls.vercel.app/

  ## Installation

  Install this plugin in the same environment as Datasette.

      $ datasette install datasette-hashed-urls

  ## Usage

  Once installed, this plugin will act on any immutable database files that are loaded into Datasette:

      datasette -i fixtures.db

  The database will automatically be renamed to incorporate a hash of the contents of the SQLite file - so the above database would be served as:

      http://127.0.0.1:8001/fixtures-aa7318b

  Every page that accesss that database, including JSON endpoints, will be served with the following far-future cache expiry header:

      cache-control: max-age=31536000, public

  Here `max-age=31536000` is the number of seconds in a year.

  A caching proxy such as Cloudflare can then be used to cache and accelerate content served by Datasette.

  When the database file is updated and the server is restarted, the hash will change and content will be served from a new URL. Any hits to the previous hashed URLs will be automatically redirected.

  If you run Datasette using the `--crossdb` option to enable [cross-database queries](https://docs.datasette.io/en/stable/sql_queries.html#cross-database-queries) the `_memory` database will also have a hash added to its URL - in this case, the hash will be a combination of the hashes of the other attached databases.

  ## Configuration

  You can use the `max_age` plugin configuration setting to change the cache duration specified in the `cache-control` HTTP header.

  To set the cache expiry time to one hour you would add this to your Datasette `metadata.json` configuration file:

  ```json
  {
      "plugins": {
          "datasette-hashed-urls": {
              "max_age": 3600
          }
      }
  }
  ```

  ## History

  This functionality used to ship as part of Datasette itself, as a feature called [Hashed URL mode](https://docs.datasette.io/en/0.60.2/performance.html#hashed-url-mode).

  That feature has been deprecated and will be removed in Datasette 1.0. This plugin should be used as an alternative.

  ## Development

  To set up this plugin locally, first checkout the code. Then create a new virtual environment:

      cd datasette-hashed-urls
      python3 -mvenv venv
      source venv/bin/activate

  Or if you are using `pipenv`:

      pipenv shell

  Now install the dependencies and test dependencies:

      pip install -e '.[test]'

  To run the tests:

      pytest

readme_html:
  <div id="readme" class="md" data-path="README.md"><article class="markdown-body entry-content container-lg" itemprop="text"><div class="markdown-heading" dir="auto"><h1 class="heading-element" dir="auto">datasette-hashed-urls</h1><a id="user-content-datasette-hashed-urls" class="anchor" aria-label="Permalink: datasette-hashed-urls" href="#datasette-hashed-urls"><svg class="octicon octicon-link" viewbox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
  <p dir="auto"><a href="https://pypi.org/project/datasette-hashed-urls/" rel="nofollow"><img src="https://img.shields.io/pypi/v/datasette-hashed-urls.svg" alt="PyPI" style="max-width: 100%;" data-camo-src="https://camo.githubusercontent.com/d3108ad504bc4f509172f547368696153a20497c1bd9d052f70f303f9774891f/68747470733a2f2f696d672e736869656c64732e696f2f707970692f762f6461746173657474652d6861736865642d75726c732e737667"></a>
  <a href="https://github.com/simonw/datasette-hashed-urls/releases"><img src="https://img.shields.io/github/v/release/simonw/datasette-hashed-urls?include_prereleases&label=changelog" alt="Changelog" style="max-width: 100%;" data-camo-src="https://camo.githubusercontent.com/9b6f49a87a56223496758cc25fab1d95f6531a99fa5b7899ace21105f5b4222a/68747470733a2f2f696d672e736869656c64732e696f2f6769746875622f762f72656c656173652f73696d6f6e772f6461746173657474652d6861736865642d75726c733f696e636c7564655f70726572656c6561736573266c6162656c3d6368616e67656c6f67"></a>
  <a href="https://github.com/simonw/datasette-hashed-urls/actions?query=workflow%3ATest"><img src="https://github.com/simonw/datasette-hashed-urls/workflows/Test/badge.svg" alt="Tests" style="max-width: 100%;"></a>
  <a href="https://github.com/simonw/datasette-hashed-urls/blob/main/LICENSE"><img src="https://img.shields.io/badge/license-Apache%202.0-blue.svg" alt="License" style="max-width: 100%;" data-camo-src="https://camo.githubusercontent.com/c355f200ea90fddaa407b6eaab303663a669248ea3ca7b1fcf77dbe04ff5f48c/68747470733a2f2f696d672e736869656c64732e696f2f62616467652f6c6963656e73652d417061636865253230322e302d626c75652e737667"></a></p>
  <p dir="auto">Optimize Datasette performance behind a caching proxy</p>
  <p dir="auto">When you open a database file in immutable mode using the <code>-i</code> option, Datasette calculates a SHA-256 hash of the contents of that file on startup.</p>
  <p dir="auto">This content hash can then optionally be used to create URLs that are guaranteed to change if the contents of the file changes in the future.</p>
  <p dir="auto">The result is pages  that can be cached indefinitely by both browsers and caching proxies - providing a significant performance boost.</p>
  <div class="markdown-heading" dir="auto"><h2 class="heading-element" dir="auto">Demo</h2><a id="user-content-demo" class="anchor" aria-label="Permalink: Demo" href="#demo"><svg class="octicon octicon-link" viewbox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
  <p dir="auto">A demo of this plugin is running at <a href="https://datasette-hashed-urls.vercel.app/" rel="nofollow">https://datasette-hashed-urls.vercel.app/</a></p>
  <div class="markdown-heading" dir="auto"><h2 class="heading-element" dir="auto">Installation</h2><a id="user-content-installation" class="anchor" aria-label="Permalink: Installation" href="#installation"><svg class="octicon octicon-link" viewbox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
  <p dir="auto">Install this plugin in the same environment as Datasette.</p>
  <div class="snippet-clipboard-content notranslate position-relative overflow-auto" data-snippet-clipboard-copy-content="$ datasette install datasette-hashed-urls"><pre class="notranslate"><code>$ datasette install datasette-hashed-urls
  </code></pre></div>
  <div class="markdown-heading" dir="auto"><h2 class="heading-element" dir="auto">Usage</h2><a id="user-content-usage" class="anchor" aria-label="Permalink: Usage" href="#usage"><svg class="octicon octicon-link" viewbox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
  <p dir="auto">Once installed, this plugin will act on any immutable database files that are loaded into Datasette:</p>
  <div class="snippet-clipboard-content notranslate position-relative overflow-auto" data-snippet-clipboard-copy-content="datasette -i fixtures.db"><pre class="notranslate"><code>datasette -i fixtures.db
  </code></pre></div>
  <p dir="auto">The database will automatically be renamed to incorporate a hash of the contents of the SQLite file - so the above database would be served as:</p>
  <div class="snippet-clipboard-content notranslate position-relative overflow-auto" data-snippet-clipboard-copy-content="http://127.0.0.1:8001/fixtures-aa7318b"><pre class="notranslate"><code>http://127.0.0.1:8001/fixtures-aa7318b
  </code></pre></div>
  <p dir="auto">Every page that accesss that database, including JSON endpoints, will be served with the following far-future cache expiry header:</p>
  <div class="snippet-clipboard-content notranslate position-relative overflow-auto" data-snippet-clipboard-copy-content="cache-control: max-age=31536000, public"><pre class="notranslate"><code>cache-control: max-age=31536000, public
  </code></pre></div>
  <p dir="auto">Here <code>max-age=31536000</code> is the number of seconds in a year.</p>
  <p dir="auto">A caching proxy such as Cloudflare can then be used to cache and accelerate content served by Datasette.</p>
  <p dir="auto">When the database file is updated and the server is restarted, the hash will change and content will be served from a new URL. Any hits to the previous hashed URLs will be automatically redirected.</p>
  <p dir="auto">If you run Datasette using the <code>--crossdb</code> option to enable <a href="https://docs.datasette.io/en/stable/sql_queries.html#cross-database-queries" rel="nofollow">cross-database queries</a> the <code>_memory</code> database will also have a hash added to its URL - in this case, the hash will be a combination of the hashes of the other attached databases.</p>
  <div class="markdown-heading" dir="auto"><h2 class="heading-element" dir="auto">Configuration</h2><a id="user-content-configuration" class="anchor" aria-label="Permalink: Configuration" href="#configuration"><svg class="octicon octicon-link" viewbox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
  <p dir="auto">You can use the <code>max_age</code> plugin configuration setting to change the cache duration specified in the <code>cache-control</code> HTTP header.</p>
  <p dir="auto">To set the cache expiry time to one hour you would add this to your Datasette <code>metadata.json</code> configuration file:</p>
  <div class="highlight highlight-source-json notranslate position-relative overflow-auto" dir="auto" data-snippet-clipboard-copy-content="{
      "plugins": {
          "datasette-hashed-urls": {
              "max_age": 3600
          }
      }
  }"><pre>{
      <span class="pl-ent">"plugins"</span>: {
          <span class="pl-ent">"datasette-hashed-urls"</span>: {
              <span class="pl-ent">"max_age"</span>: <span class="pl-c1">3600</span>
          }
      }
  }</pre></div>
  <div class="markdown-heading" dir="auto"><h2 class="heading-element" dir="auto">History</h2><a id="user-content-history" class="anchor" aria-label="Permalink: History" href="#history"><svg class="octicon octicon-link" viewbox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
  <p dir="auto">This functionality used to ship as part of Datasette itself, as a feature called <a href="https://docs.datasette.io/en/0.60.2/performance.html#hashed-url-mode" rel="nofollow">Hashed URL mode</a>.</p>
  <p dir="auto">That feature has been deprecated and will be removed in Datasette 1.0. This plugin should be used as an alternative.</p>
  <div class="markdown-heading" dir="auto"><h2 class="heading-element" dir="auto">Development</h2><a id="user-content-development" class="anchor" aria-label="Permalink: Development" href="#development"><svg class="octicon octicon-link" viewbox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
  <p dir="auto">To set up this plugin locally, first checkout the code. Then create a new virtual environment:</p>
  <div class="snippet-clipboard-content notranslate position-relative overflow-auto" data-snippet-clipboard-copy-content="cd datasette-hashed-urls
  python3 -mvenv venv
  source venv/bin/activate"><pre class="notranslate"><code>cd datasette-hashed-urls
  python3 -mvenv venv
  source venv/bin/activate
  </code></pre></div>
  <p dir="auto">Or if you are using <code>pipenv</code>:</p>
  <div class="snippet-clipboard-content notranslate position-relative overflow-auto" data-snippet-clipboard-copy-content="pipenv shell"><pre class="notranslate"><code>pipenv shell
  </code></pre></div>
  <p dir="auto">Now install the dependencies and test dependencies:</p>
  <div class="snippet-clipboard-content notranslate position-relative overflow-auto" data-snippet-clipboard-copy-content="pip install -e '.[test]'"><pre class="notranslate"><code>pip install -e '.[test]'
  </code></pre></div>
  <p dir="auto">To run the tests:</p>
  <div class="snippet-clipboard-content notranslate position-relative overflow-auto" data-snippet-clipboard-copy-content="pytest"><pre class="notranslate"><code>pytest
  </code></pre></div>
  </article></div>


node_id: R_kgDOHENadA
name: datasette-packages
full_name: simonw/datasette-packages
html_url: https://github.com/simonw/datasette-packages
description: Show a list of currently installed Python packages
created_at: 2022-03-25T22:04:23Z
updated_at: 2026-02-08T18:13:53Z
pushed_at: 2026-02-08T18:16:31Z
language: Python
license: apache-2.0
topics: []
visibility: public
default_branch: main
permissions: {"admin": false, "maintain": false, "push": false, "triage": false, "pull": false}
pull_request_creation_policy: all
readme:
  # datasette-packages

  [![PyPI](https://img.shields.io/pypi/v/datasette-packages.svg)](https://pypi.org/project/datasette-packages/)
  [![Changelog](https://img.shields.io/github/v/release/simonw/datasette-packages?include_prereleases&label=changelog)](https://github.com/simonw/datasette-packages/releases)
  [![Tests](https://github.com/simonw/datasette-packages/workflows/Test/badge.svg)](https://github.com/simonw/datasette-packages/actions?query=workflow%3ATest)
  [![License](https://img.shields.io/badge/license-Apache%202.0-blue.svg)](https://github.com/simonw/datasette-packages/blob/main/LICENSE)

  Show a list of currently installed Python packages

  ## Installation

  Install this plugin in the same environment as Datasette.

      $ datasette install datasette-packages

  ## Usage

  Visit `/-/packages` to see a list of installed Python packages.

  Visit `/-/packages.json` to get that back as JSON.

  ## Demo

  The output of this plugin can be seen here:

  - https://latest-with-plugins.datasette.io/-/packages
  - https://latest-with-plugins.datasette.io/-/packages.json

  ## With datasette-graphql

  if you have version 2.1 or higher of the [datasette-graphql](https://datasette.io/plugins/datasette-graphql) plugin installed you can also query the list of packages using this GraphQL query:

  ```graphql
  {
    packages {
      name
      version
    }
  }
  ```
  [Demo of this query](https://latest-with-plugins.datasette.io/graphql?query=%7B%0A%20%20packages%20%7B%0A%20%20%20%20name%0A%20%20%20%20version%0A%20%20%7D%0A%7D).

  ## Development

  To set up this plugin locally, first checkout the code. Then create a new virtual environment:

      cd datasette-packages
      python3 -mvenv venv
      source venv/bin/activate

  Now install the dependencies and test dependencies:

      pip install -e '.[test]'

  To run the tests:

      pytest

readme_html:
  <div id="readme" class="md" data-path="README.md"><article class="markdown-body entry-content container-lg" itemprop="text"><div class="markdown-heading" dir="auto"><h1 class="heading-element" dir="auto">datasette-packages</h1><a id="user-content-datasette-packages" class="anchor" aria-label="Permalink: datasette-packages" href="#datasette-packages"><svg class="octicon octicon-link" viewbox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
  <p dir="auto"><a href="https://pypi.org/project/datasette-packages/" rel="nofollow"><img src="https://img.shields.io/pypi/v/datasette-packages.svg" alt="PyPI" style="max-width: 100%;" data-camo-src="https://camo.githubusercontent.com/22927ea181246d92b960b01c9cd5ab67e9e12442e2359cc9dff1242e311d05b7/68747470733a2f2f696d672e736869656c64732e696f2f707970692f762f6461746173657474652d7061636b616765732e737667"></a>
  <a href="https://github.com/simonw/datasette-packages/releases"><img src="https://img.shields.io/github/v/release/simonw/datasette-packages?include_prereleases&label=changelog" alt="Changelog" style="max-width: 100%;" data-camo-src="https://camo.githubusercontent.com/29d947a4900ebc7859594a091c2e3337af36f81faf465d7d73cfabea3a8f8b1d/68747470733a2f2f696d672e736869656c64732e696f2f6769746875622f762f72656c656173652f73696d6f6e772f6461746173657474652d7061636b616765733f696e636c7564655f70726572656c6561736573266c6162656c3d6368616e67656c6f67"></a>
  <a href="https://github.com/simonw/datasette-packages/actions?query=workflow%3ATest"><img src="https://github.com/simonw/datasette-packages/workflows/Test/badge.svg" alt="Tests" style="max-width: 100%;"></a>
  <a href="https://github.com/simonw/datasette-packages/blob/main/LICENSE"><img src="https://img.shields.io/badge/license-Apache%202.0-blue.svg" alt="License" style="max-width: 100%;" data-camo-src="https://camo.githubusercontent.com/c355f200ea90fddaa407b6eaab303663a669248ea3ca7b1fcf77dbe04ff5f48c/68747470733a2f2f696d672e736869656c64732e696f2f62616467652f6c6963656e73652d417061636865253230322e302d626c75652e737667"></a></p>
  <p dir="auto">Show a list of currently installed Python packages</p>
  <div class="markdown-heading" dir="auto"><h2 class="heading-element" dir="auto">Installation</h2><a id="user-content-installation" class="anchor" aria-label="Permalink: Installation" href="#installation"><svg class="octicon octicon-link" viewbox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
  <p dir="auto">Install this plugin in the same environment as Datasette.</p>
  <div class="snippet-clipboard-content notranslate position-relative overflow-auto" data-snippet-clipboard-copy-content="$ datasette install datasette-packages"><pre class="notranslate"><code>$ datasette install datasette-packages
  </code></pre></div>
  <div class="markdown-heading" dir="auto"><h2 class="heading-element" dir="auto">Usage</h2><a id="user-content-usage" class="anchor" aria-label="Permalink: Usage" href="#usage"><svg class="octicon octicon-link" viewbox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
  <p dir="auto">Visit <code>/-/packages</code> to see a list of installed Python packages.</p>
  <p dir="auto">Visit <code>/-/packages.json</code> to get that back as JSON.</p>
  <div class="markdown-heading" dir="auto"><h2 class="heading-element" dir="auto">Demo</h2><a id="user-content-demo" class="anchor" aria-label="Permalink: Demo" href="#demo"><svg class="octicon octicon-link" viewbox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
  <p dir="auto">The output of this plugin can be seen here:</p>
  <ul dir="auto">
  <li><a href="https://latest-with-plugins.datasette.io/-/packages" rel="nofollow">https://latest-with-plugins.datasette.io/-/packages</a></li>
  <li><a href="https://latest-with-plugins.datasette.io/-/packages.json" rel="nofollow">https://latest-with-plugins.datasette.io/-/packages.json</a></li>
  </ul>
  <div class="markdown-heading" dir="auto"><h2 class="heading-element" dir="auto">With datasette-graphql</h2><a id="user-content-with-datasette-graphql" class="anchor" aria-label="Permalink: With datasette-graphql" href="#with-datasette-graphql"><svg class="octicon octicon-link" viewbox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
  <p dir="auto">if you have version 2.1 or higher of the <a href="https://datasette.io/plugins/datasette-graphql" rel="nofollow">datasette-graphql</a> plugin installed you can also query the list of packages using this GraphQL query:</p>
  <div class="highlight highlight-source-graphql notranslate position-relative overflow-auto" dir="auto" data-snippet-clipboard-copy-content="{
    packages {
      name
      version
    }
  }"><pre>{
    <span class="pl-v">packages</span> {
      <span class="pl-v">name</span>
      <span class="pl-v">version</span>
    }
  }</pre></div>
  <p dir="auto"><a href="https://latest-with-plugins.datasette.io/graphql?query=%7B%0A%20%20packages%20%7B%0A%20%20%20%20name%0A%20%20%20%20version%0A%20%20%7D%0A%7D" rel="nofollow">Demo of this query</a>.</p>
  <div class="markdown-heading" dir="auto"><h2 class="heading-element" dir="auto">Development</h2><a id="user-content-development" class="anchor" aria-label="Permalink: Development" href="#development"><svg class="octicon octicon-link" viewbox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
  <p dir="auto">To set up this plugin locally, first checkout the code. Then create a new virtual environment:</p>
  <div class="snippet-clipboard-content notranslate position-relative overflow-auto" data-snippet-clipboard-copy-content="cd datasette-packages
  python3 -mvenv venv
  source venv/bin/activate"><pre class="notranslate"><code>cd datasette-packages
  python3 -mvenv venv
  source venv/bin/activate
  </code></pre></div>
  <p dir="auto">Now install the dependencies and test dependencies:</p>
  <div class="snippet-clipboard-content notranslate position-relative overflow-auto" data-snippet-clipboard-copy-content="pip install -e '.[test]'"><pre class="notranslate"><code>pip install -e '.[test]'
  </code></pre></div>
  <p dir="auto">To run the tests:</p>
  <div class="snippet-clipboard-content notranslate position-relative overflow-auto" data-snippet-clipboard-copy-content="pytest"><pre class="notranslate"><code>pytest
  </code></pre></div>
  </article></div>


node_id: R_kgDOHEfRqA
name: datasette-auth0
full_name: simonw/datasette-auth0
html_url: https://github.com/simonw/datasette-auth0
description: Datasette plugin that authenticates users using Auth0
created_at: 2022-03-26T21:19:31Z
updated_at: 2023-04-05T04:53:07Z
pushed_at: 2022-03-28T03:04:52Z
language: Python
license: apache-2.0
topics: ["auth0", "datasette", "datasette-plugin"]
visibility: public
default_branch: main
permissions: {"admin": false, "maintain": false, "push": false, "triage": false, "pull": true}
readme:
  # datasette-auth0

  [![PyPI](https://img.shields.io/pypi/v/datasette-auth0.svg)](https://pypi.org/project/datasette-auth0/)
  [![Changelog](https://img.shields.io/github/v/release/simonw/datasette-auth0?include_prereleases&label=changelog)](https://github.com/simonw/datasette-auth0/releases)
  [![Tests](https://github.com/simonw/datasette-auth0/workflows/Test/badge.svg)](https://github.com/simonw/datasette-auth0/actions?query=workflow%3ATest)
  [![License](https://img.shields.io/badge/license-Apache%202.0-blue.svg)](https://github.com/simonw/datasette-auth0/blob/main/LICENSE)

  Datasette plugin that authenticates users using [Auth0](https://auth0.com/)

  See [Simplest possible OAuth authentication with Auth0](https://til.simonwillison.net/auth0/oauth-with-auth0) for more about how this plugin works.

  ## Installation

  Install this plugin in the same environment as Datasette.

      $ datasette install datasette-auth0

  ## Demo

  You can try this out at [datasette-auth0-demo.datasette.io](https://datasette-auth0-demo.datasette.io/) - click on the top right menu icon and select "Sign in with Auth0".

  ## Initial configuration

  First, create a new application in Auth0. You will need the domain, client ID and client secret for that application.

  The domain should be something like `mysite.us.auth0.com`.

  Add `http://127.0.0.1:8001/-/auth0-callback` to the list of Allowed Callback URLs.

  Then configure these plugin secrets using `metadata.yml`:

  ```yaml
  plugins:
    datasette-auth0:
      domain:
        "$env": AUTH0_DOMAIN
      client_id:
        "$env": AUTH0_CLIENT_ID
      client_secret:
        "$env": AUTH0_CLIENT_SECRET
  ```
  Only the `client_secret` needs to be kept secret, but for consistency I recommend using the `$env` mechanism for all three.

  In development, you can run Datasette and pass in environment variables like this:
  ```
  AUTH0_DOMAIN="your-domain.us.auth0.com" \
  AUTH0_CLIENT_ID="...client-id-goes-here..." \
  AUTH0_CLIENT_SECRET="...secret-goes-here..." \
  datasette -m metadata.yml
  ```

  If you are deploying using `datasette publish` you can pass these using `--plugin-secret`. For example, to deploy using Cloud Run you might run the following:
  ```
  datasette publish cloudrun mydatabase.db \
  --install datasette-auth0 \
  --plugin-secret datasette-auth0 domain "your-domain.us.auth0.com" \
  --plugin-secret datasette-auth0 client_id "your-client-id" \
  --plugin-secret datasette-auth0 client_secret "your-client-secret" \
  --service datasette-auth0-demo
  ```
  Once your Datasette instance is deployed, you will need to add its callback URL to the "Allowed Callback URLs" list in Auth0.

  The callback URL should be something like:

      https://url-to-your-datasette/-/auth0-callback

  ## Usage

  Once installed, a "Sign in with Auth0" menu item will appear in the Datasette main menu.

  You can sign in and then visit the `/-/actor` page to see full details of the `auth0` profile that has been authenticated.

  You can then use [Datasette permissions](https://docs.datasette.io/en/stable/authentication.html#configuring-permissions-in-metadata-json) to grant or deny access to different parts of Datasette based on the authenticated user.

  ## Development

  To set up this plugin locally, first checkout the code. Then create a new virtual environment:

      cd datasette-auth0
      python3 -mvenv venv
      source venv/bin/activate

  Now install the dependencies and test dependencies:

      pip install -e '.[test]'

  To run the tests:

      pytest

readme_html:
  <div id="readme" class="md" data-path="README.md"><article class="markdown-body entry-content container-lg" itemprop="text"><div class="markdown-heading" dir="auto"><h1 class="heading-element" dir="auto">datasette-auth0</h1><a id="user-content-datasette-auth0" class="anchor" aria-label="Permalink: datasette-auth0" href="#datasette-auth0"><svg class="octicon octicon-link" viewbox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
  <p dir="auto"><a href="https://pypi.org/project/datasette-auth0/" rel="nofollow"><img src="https://img.shields.io/pypi/v/datasette-auth0.svg" alt="PyPI" style="max-width: 100%;" data-camo-src="https://camo.githubusercontent.com/b7255678ab31c5caf64d29803f7751478ac054584b478569a6de2819941d317b/68747470733a2f2f696d672e736869656c64732e696f2f707970692f762f6461746173657474652d61757468302e737667"></a>
  <a href="https://github.com/simonw/datasette-auth0/releases"><img src="https://img.shields.io/github/v/release/simonw/datasette-auth0?include_prereleases&label=changelog" alt="Changelog" style="max-width: 100%;" data-camo-src="https://camo.githubusercontent.com/46853a03d3975d5745fed896739a050826d5487b7b79a5b48c8db9d77a33cb18/68747470733a2f2f696d672e736869656c64732e696f2f6769746875622f762f72656c656173652f73696d6f6e772f6461746173657474652d61757468303f696e636c7564655f70726572656c6561736573266c6162656c3d6368616e67656c6f67"></a>
  <a href="https://github.com/simonw/datasette-auth0/actions?query=workflow%3ATest"><img src="https://github.com/simonw/datasette-auth0/workflows/Test/badge.svg" alt="Tests" style="max-width: 100%;"></a>
  <a href="https://github.com/simonw/datasette-auth0/blob/main/LICENSE"><img src="https://img.shields.io/badge/license-Apache%202.0-blue.svg" alt="License" style="max-width: 100%;" data-camo-src="https://camo.githubusercontent.com/c355f200ea90fddaa407b6eaab303663a669248ea3ca7b1fcf77dbe04ff5f48c/68747470733a2f2f696d672e736869656c64732e696f2f62616467652f6c6963656e73652d417061636865253230322e302d626c75652e737667"></a></p>
  <p dir="auto">Datasette plugin that authenticates users using <a href="https://auth0.com/" rel="nofollow">Auth0</a></p>
  <p dir="auto">See <a href="https://til.simonwillison.net/auth0/oauth-with-auth0" rel="nofollow">Simplest possible OAuth authentication with Auth0</a> for more about how this plugin works.</p>
  <div class="markdown-heading" dir="auto"><h2 class="heading-element" dir="auto">Installation</h2><a id="user-content-installation" class="anchor" aria-label="Permalink: Installation" href="#installation"><svg class="octicon octicon-link" viewbox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
  <p dir="auto">Install this plugin in the same environment as Datasette.</p>
  <div class="snippet-clipboard-content notranslate position-relative overflow-auto" data-snippet-clipboard-copy-content="$ datasette install datasette-auth0"><pre class="notranslate"><code>$ datasette install datasette-auth0
  </code></pre></div>
  <div class="markdown-heading" dir="auto"><h2 class="heading-element" dir="auto">Demo</h2><a id="user-content-demo" class="anchor" aria-label="Permalink: Demo" href="#demo"><svg class="octicon octicon-link" viewbox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
  <p dir="auto">You can try this out at <a href="https://datasette-auth0-demo.datasette.io/" rel="nofollow">datasette-auth0-demo.datasette.io</a> - click on the top right menu icon and select "Sign in with Auth0".</p>
  <div class="markdown-heading" dir="auto"><h2 class="heading-element" dir="auto">Initial configuration</h2><a id="user-content-initial-configuration" class="anchor" aria-label="Permalink: Initial configuration" href="#initial-configuration"><svg class="octicon octicon-link" viewbox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
  <p dir="auto">First, create a new application in Auth0. You will need the domain, client ID and client secret for that application.</p>
  <p dir="auto">The domain should be something like <code>mysite.us.auth0.com</code>.</p>
  <p dir="auto">Add <code>http://127.0.0.1:8001/-/auth0-callback</code> to the list of Allowed Callback URLs.</p>
  <p dir="auto">Then configure these plugin secrets using <code>metadata.yml</code>:</p>
  <div class="highlight highlight-source-yaml notranslate position-relative overflow-auto" dir="auto" data-snippet-clipboard-copy-content="plugins:
    datasette-auth0:
      domain:
        "$env": AUTH0_DOMAIN
      client_id:
        "$env": AUTH0_CLIENT_ID
      client_secret:
        "$env": AUTH0_CLIENT_SECRET"><pre><span class="pl-ent">plugins</span>:
    <span class="pl-ent">datasette-auth0</span>:
      <span class="pl-ent">domain</span>:
        <span class="pl-s"><span class="pl-pds">"</span><span class="pl-ent">$env</span><span class="pl-pds">"</span></span>: <span class="pl-s">AUTH0_DOMAIN</span>
      <span class="pl-ent">client_id</span>:
        <span class="pl-s"><span class="pl-pds">"</span><span class="pl-ent">$env</span><span class="pl-pds">"</span></span>: <span class="pl-s">AUTH0_CLIENT_ID</span>
      <span class="pl-ent">client_secret</span>:
        <span class="pl-s"><span class="pl-pds">"</span><span class="pl-ent">$env</span><span class="pl-pds">"</span></span>: <span class="pl-s">AUTH0_CLIENT_SECRET</span></pre></div>
  <p dir="auto">Only the <code>client_secret</code> needs to be kept secret, but for consistency I recommend using the <code>$env</code> mechanism for all three.</p>
  <p dir="auto">In development, you can run Datasette and pass in environment variables like this:</p>
  <div class="snippet-clipboard-content notranslate position-relative overflow-auto" data-snippet-clipboard-copy-content="AUTH0_DOMAIN="your-domain.us.auth0.com" \
  AUTH0_CLIENT_ID="...client-id-goes-here..." \
  AUTH0_CLIENT_SECRET="...secret-goes-here..." \
  datasette -m metadata.yml"><pre class="notranslate"><code>AUTH0_DOMAIN="your-domain.us.auth0.com" \
  AUTH0_CLIENT_ID="...client-id-goes-here..." \
  AUTH0_CLIENT_SECRET="...secret-goes-here..." \
  datasette -m metadata.yml
  </code></pre></div>
  <p dir="auto">If you are deploying using <code>datasette publish</code> you can pass these using <code>--plugin-secret</code>. For example, to deploy using Cloud Run you might run the following:</p>
  <div class="snippet-clipboard-content notranslate position-relative overflow-auto" data-snippet-clipboard-copy-content="datasette publish cloudrun mydatabase.db \
  --install datasette-auth0 \
  --plugin-secret datasette-auth0 domain "your-domain.us.auth0.com" \
  --plugin-secret datasette-auth0 client_id "your-client-id" \
  --plugin-secret datasette-auth0 client_secret "your-client-secret" \
  --service datasette-auth0-demo"><pre class="notranslate"><code>datasette publish cloudrun mydatabase.db \
  --install datasette-auth0 \
  --plugin-secret datasette-auth0 domain "your-domain.us.auth0.com" \
  --plugin-secret datasette-auth0 client_id "your-client-id" \
  --plugin-secret datasette-auth0 client_secret "your-client-secret" \
  --service datasette-auth0-demo
  </code></pre></div>
  <p dir="auto">Once your Datasette instance is deployed, you will need to add its callback URL to the "Allowed Callback URLs" list in Auth0.</p>
  <p dir="auto">The callback URL should be something like:</p>
  <div class="snippet-clipboard-content notranslate position-relative overflow-auto" data-snippet-clipboard-copy-content="https://url-to-your-datasette/-/auth0-callback"><pre class="notranslate"><code>https://url-to-your-datasette/-/auth0-callback
  </code></pre></div>
  <div class="markdown-heading" dir="auto"><h2 class="heading-element" dir="auto">Usage</h2><a id="user-content-usage" class="anchor" aria-label="Permalink: Usage" href="#usage"><svg class="octicon octicon-link" viewbox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
  <p dir="auto">Once installed, a "Sign in with Auth0" menu item will appear in the Datasette main menu.</p>
  <p dir="auto">You can sign in and then visit the <code>/-/actor</code> page to see full details of the <code>auth0</code> profile that has been authenticated.</p>
  <p dir="auto">You can then use <a href="https://docs.datasette.io/en/stable/authentication.html#configuring-permissions-in-metadata-json" rel="nofollow">Datasette permissions</a> to grant or deny access to different parts of Datasette based on the authenticated user.</p>
  <div class="markdown-heading" dir="auto"><h2 class="heading-element" dir="auto">Development</h2><a id="user-content-development" class="anchor" aria-label="Permalink: Development" href="#development"><svg class="octicon octicon-link" viewbox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
  <p dir="auto">To set up this plugin locally, first checkout the code. Then create a new virtual environment:</p>
  <div class="snippet-clipboard-content notranslate position-relative overflow-auto" data-snippet-clipboard-copy-content="cd datasette-auth0
  python3 -mvenv venv
  source venv/bin/activate"><pre class="notranslate"><code>cd datasette-auth0
  python3 -mvenv venv
  source venv/bin/activate
  </code></pre></div>
  <p dir="auto">Now install the dependencies and test dependencies:</p>
  <div class="snippet-clipboard-content notranslate position-relative overflow-auto" data-snippet-clipboard-copy-content="pip install -e '.[test]'"><pre class="notranslate"><code>pip install -e '.[test]'
  </code></pre></div>
  <p dir="auto">To run the tests:</p>
  <div class="snippet-clipboard-content notranslate position-relative overflow-auto" data-snippet-clipboard-copy-content="pytest"><pre class="notranslate"><code>pytest
  </code></pre></div>
  </article></div>


node_id: R_kgDOHEflbA
name: datasette-nteract-data-explorer
full_name: hydrosquall/datasette-nteract-data-explorer
html_url: https://github.com/hydrosquall/datasette-nteract-data-explorer
description: automatic visual data explorer for datasette
created_at: 2022-03-26T21:47:17Z
updated_at: 2024-11-19T13:52:18Z
pushed_at: 2023-04-20T03:33:51Z
homepage: https://datasette-nteract-data-explorer.vercel.app/
language: TypeScript
license: apache-2.0
topics: ["automatic-viz", "datasette", "datasette-plugin", "dataviz"]
visibility: public
default_branch: main
permissions: {"admin": false, "maintain": false, "push": false, "triage": false, "pull": true}
readme:
  # datasette-nteract-data-explorer

  [![PyPI](https://img.shields.io/pypi/v/datasette-nteract-data-explorer.svg)](https://pypi.org/project/datasette-nteract-data-explorer/)
  [![Changelog](https://img.shields.io/github/v/release/hydrosquall/datasette-nteract-data-explorer?include_prereleases&label=changelog)](https://github.com/hydrosquall/datasette-nteract-data-explorer/releases)
  [![Tests](https://github.com/hydrosquall/datasette-nteract-data-explorer/workflows/Test/badge.svg)](https://github.com/hydrosquall/datasette-nteract-data-explorer/actions?query=workflow%3ATest)
  [![License](https://img.shields.io/badge/license-Apache%202.0-blue.svg)](https://github.com/hydrosquall/datasette-nteract-data-explorer/blob/main/LICENSE)

  An automatic data visualization plugin for the [Datasette](https://datasette.io/) ecosystem. See your dataset from multiple views with an easy-to-use, customizable menu-based interface.

  ## Demo

  Try the [live demo](https://datasette-nteract-data-explorer.vercel.app/happy_planet_index/hpi_cleaned?_size=137)

  ![screenshot](https://p-qkfgo2.t2.n0.cdn.getcloudapp.com/items/yAuK9LRE/6802f849-315d-4a21-93b4-61c94d066bdc.jpg?v=f1ceee5ed70832d74e745b6508baeffb)

  _Running Datasette with the Happy Planet Index dataset_

  ## Installation

  Install this plugin in the same Python environment as Datasette.

  ```bash
  datasette install datasette-nteract-data-explorer
  ```

  ## Usage

  - Click "View in Data Explorer" to expand the visualization panel
  - Click the icons on the right side to change the visualization type.
  - Use the menus underneath the graphing area to configure your graph (e.g. change which columns to graph, colors to use, etc)
  - Use "advanced settings" mode to override the inferred column types. For example, you may want to treat a number as a "string" to be able to use it as a category.
  - See a [live demo](https://data-explorer.nteract.io/) of the original Nteract data-explorer component used in isolation.

  You can run a minimal demo after the installation step

  ```bash
  datasette -i demo/happy_planet_index.db
  ```

  If you're interested in improving the demo site, you can run a copy of the site the extra metadata/plugins used in the [published demo](https://datasette-nteract-data-explorer.vercel.app).

  ```bash
  make run-demo
  ```

  Thank you for reading this far! If you use the Data Explorer in your own site and would like others to find it, you can [mention it here](https://github.com/hydrosquall/datasette-nteract-data-explorer/discussions/10).

  ## Development

  See [contributing docs](./docs/CONTRIBUTING.md).

  ## Acknowledgements

  - The [Data Explorer](https://github.com/nteract/data-explorer) was designed by Elijah Meeks. I co-maintain this project as part of the [Nteract](https://nteract.io/) open-source team. You can read about the design behind this tool [here](https://blog.nteract.io/designing-the-nteract-data-explorer-f4476d53f897)
  - The data model is based on the [Frictionless Data Spec](https://specs.frictionlessdata.io/).
  - This plugin was bootstrapped by Simon Willison's [Datasette plugin template](https://simonwillison.net/2020/Jun/20/cookiecutter-plugins/)
  - Demo dataset from the [Happy Planet Index](https://happyplanetindex.org/) was cleaned by Doris Lee. This dataset was chosen because of its global appeal, modest size, and variety in column datatypes (numbers, low cardinality and high cardinality strings, booleans).
  - Hosting for the demo site is provided by Vercel.

  [![site hosted by vercel](https://www.datocms-assets.com/31049/1618983297-powered-by-vercel.svg)](https://vercel.com/?utm_source=datasette-visualization-plugin-demos&utm_campaign=oss)

readme_html:
  <div id="readme" class="md" data-path="README.md"><article class="markdown-body entry-content container-lg" itemprop="text"><div class="markdown-heading" dir="auto"><h1 class="heading-element" dir="auto">datasette-nteract-data-explorer</h1><a id="user-content-datasette-nteract-data-explorer" class="anchor" aria-label="Permalink: datasette-nteract-data-explorer" href="#datasette-nteract-data-explorer"><svg class="octicon octicon-link" viewbox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
  <p dir="auto"><a href="https://pypi.org/project/datasette-nteract-data-explorer/" rel="nofollow"><img src="https://img.shields.io/pypi/v/datasette-nteract-data-explorer.svg" alt="PyPI" style="max-width: 100%;" data-camo-src="https://camo.githubusercontent.com/e5b51360f63c0406283373b1f02ceed4e9d3200c9b0368b8fde2f61bf6e0dbb6/68747470733a2f2f696d672e736869656c64732e696f2f707970692f762f6461746173657474652d6e7465726163742d646174612d6578706c6f7265722e737667"></a>
  <a href="https://github.com/hydrosquall/datasette-nteract-data-explorer/releases"><img src="https://img.shields.io/github/v/release/hydrosquall/datasette-nteract-data-explorer?include_prereleases&label=changelog" alt="Changelog" style="max-width: 100%;" data-camo-src="https://camo.githubusercontent.com/1904790827eca9879090116e9199830c67407d9b9132091d854f4e31181eb554/68747470733a2f2f696d672e736869656c64732e696f2f6769746875622f762f72656c656173652f687964726f737175616c6c2f6461746173657474652d6e7465726163742d646174612d6578706c6f7265723f696e636c7564655f70726572656c6561736573266c6162656c3d6368616e67656c6f67"></a>
  <a href="https://github.com/hydrosquall/datasette-nteract-data-explorer/actions?query=workflow%3ATest"><img src="https://github.com/hydrosquall/datasette-nteract-data-explorer/workflows/Test/badge.svg" alt="Tests" style="max-width: 100%;"></a>
  <a href="https://github.com/hydrosquall/datasette-nteract-data-explorer/blob/main/LICENSE"><img src="https://img.shields.io/badge/license-Apache%202.0-blue.svg" alt="License" style="max-width: 100%;" data-camo-src="https://camo.githubusercontent.com/c355f200ea90fddaa407b6eaab303663a669248ea3ca7b1fcf77dbe04ff5f48c/68747470733a2f2f696d672e736869656c64732e696f2f62616467652f6c6963656e73652d417061636865253230322e302d626c75652e737667"></a></p>
  <p dir="auto">An automatic data visualization plugin for the <a href="https://datasette.io/" rel="nofollow">Datasette</a> ecosystem. See your dataset from multiple views with an easy-to-use, customizable menu-based interface.</p>
  <div class="markdown-heading" dir="auto"><h2 class="heading-element" dir="auto">Demo</h2><a id="user-content-demo" class="anchor" aria-label="Permalink: Demo" href="#demo"><svg class="octicon octicon-link" viewbox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
  <p dir="auto">Try the <a href="https://datasette-nteract-data-explorer.vercel.app/happy_planet_index/hpi_cleaned?_size=137" rel="nofollow">live demo</a></p>
  <p dir="auto"><a target="_blank" rel="noopener noreferrer nofollow" href="https://camo.githubusercontent.com/8adda58b629b88f138f7cbc7aa0e244ed8dd86ccf2199d581d1e5e355e16d4c0/68747470733a2f2f702d716b66676f322e74322e6e302e63646e2e676574636c6f75646170702e636f6d2f6974656d732f7941754b394c52452f36383032663834392d333135642d346132312d393362342d3631633934643036366264632e6a70673f763d6631636565653565643730383332643734653734356236353038626165666662"><img src="https://p-qkfgo2.t2.n0.cdn.getcloudapp.com/items/yAuK9LRE/6802f849-315d-4a21-93b4-61c94d066bdc.jpg?v=f1ceee5ed70832d74e745b6508baeffb" alt="screenshot" style="max-width: 100%;" data-camo-src="https://camo.githubusercontent.com/8adda58b629b88f138f7cbc7aa0e244ed8dd86ccf2199d581d1e5e355e16d4c0/68747470733a2f2f702d716b66676f322e74322e6e302e63646e2e676574636c6f75646170702e636f6d2f6974656d732f7941754b394c52452f36383032663834392d333135642d346132312d393362342d3631633934643036366264632e6a70673f763d6631636565653565643730383332643734653734356236353038626165666662"></a></p>
  <p dir="auto"><em>Running Datasette with the Happy Planet Index dataset</em></p>
  <div class="markdown-heading" dir="auto"><h2 class="heading-element" dir="auto">Installation</h2><a id="user-content-installation" class="anchor" aria-label="Permalink: Installation" href="#installation"><svg class="octicon octicon-link" viewbox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
  <p dir="auto">Install this plugin in the same Python environment as Datasette.</p>
  <div class="highlight highlight-source-shell notranslate position-relative overflow-auto" dir="auto" data-snippet-clipboard-copy-content="datasette install datasette-nteract-data-explorer"><pre>datasette install datasette-nteract-data-explorer</pre></div>
  <div class="markdown-heading" dir="auto"><h2 class="heading-element" dir="auto">Usage</h2><a id="user-content-usage" class="anchor" aria-label="Permalink: Usage" href="#usage"><svg class="octicon octicon-link" viewbox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
  <ul dir="auto">
  <li>Click "View in Data Explorer" to expand the visualization panel</li>
  <li>Click the icons on the right side to change the visualization type.</li>
  <li>Use the menus underneath the graphing area to configure your graph (e.g. change which columns to graph, colors to use, etc)</li>
  <li>Use "advanced settings" mode to override the inferred column types. For example, you may want to treat a number as a "string" to be able to use it as a category.</li>
  <li>See a <a href="https://data-explorer.nteract.io/" rel="nofollow">live demo</a> of the original Nteract data-explorer component used in isolation.</li>
  </ul>
  <p dir="auto">You can run a minimal demo after the installation step</p>
  <div class="highlight highlight-source-shell notranslate position-relative overflow-auto" dir="auto" data-snippet-clipboard-copy-content="datasette -i demo/happy_planet_index.db"><pre>datasette -i demo/happy_planet_index.db</pre></div>
  <p dir="auto">If you're interested in improving the demo site, you can run a copy of the site the extra metadata/plugins used in the <a href="https://datasette-nteract-data-explorer.vercel.app" rel="nofollow">published demo</a>.</p>
  <div class="highlight highlight-source-shell notranslate position-relative overflow-auto" dir="auto" data-snippet-clipboard-copy-content="make run-demo"><pre>make run-demo</pre></div>
  <p dir="auto">Thank you for reading this far! If you use the Data Explorer in your own site and would like others to find it, you can <a href="https://github.com/hydrosquall/datasette-nteract-data-explorer/discussions/10" data-hovercard-type="discussion" data-hovercard-url="/hydrosquall/datasette-nteract-data-explorer/discussions/10/hovercard">mention it here</a>.</p>
  <div class="markdown-heading" dir="auto"><h2 class="heading-element" dir="auto">Development</h2><a id="user-content-development" class="anchor" aria-label="Permalink: Development" href="#development"><svg class="octicon octicon-link" viewbox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
  <p dir="auto">See <a href="./docs/CONTRIBUTING.md">contributing docs</a>.</p>
  <div class="markdown-heading" dir="auto"><h2 class="heading-element" dir="auto">Acknowledgements</h2><a id="user-content-acknowledgements" class="anchor" aria-label="Permalink: Acknowledgements" href="#acknowledgements"><svg class="octicon octicon-link" viewbox="0 0 16 16" version="1.1" width="16" height="16" aria-hidden="true"><path d="m7.775 3.275 1.25-1.25a3.5 3.5 0 1 1 4.95 4.95l-2.5 2.5a3.5 3.5 0 0 1-4.95 0 .751.751 0 0 1 .018-1.042.751.751 0 0 1 1.042-.018 1.998 1.998 0 0 0 2.83 0l2.5-2.5a2.002 2.002 0 0 0-2.83-2.83l-1.25 1.25a.751.751 0 0 1-1.042-.018.751.751 0 0 1-.018-1.042Zm-4.69 9.64a1.998 1.998 0 0 0 2.83 0l1.25-1.25a.751.751 0 0 1 1.042.018.751.751 0 0 1 .018 1.042l-1.25 1.25a3.5 3.5 0 1 1-4.95-4.95l2.5-2.5a3.5 3.5 0 0 1 4.95 0 .751.751 0 0 1-.018 1.042.751.751 0 0 1-1.042.018 1.998 1.998 0 0 0-2.83 0l-2.5 2.5a1.998 1.998 0 0 0 0 2.83Z"></path></svg></a></div>
  <ul dir="auto">
  <li>The <a href="https://github.com/nteract/data-explorer">Data Explorer</a> was designed by Elijah Meeks. I co-maintain this project as part of the <a href="https://nteract.io/" rel="nofollow">Nteract</a> open-source team. You can read about the design behind this tool <a href="https://blog.nteract.io/designing-the-nteract-data-explorer-f4476d53f897" rel="nofollow">here</a></li>
  <li>The data model is based on the <a href="https://specs.frictionlessdata.io/" rel="nofollow">Frictionless Data Spec</a>.</li>
  <li>This plugin was bootstrapped by Simon Willison's <a href="https://simonwillison.net/2020/Jun/20/cookiecutter-plugins/" rel="nofollow">Datasette plugin template</a></li>
  <li>Demo dataset from the <a href="https://happyplanetindex.org/" rel="nofollow">Happy Planet Index</a> was cleaned by Doris Lee. This dataset was chosen because of its global appeal, modest size, and variety in column datatypes (numbers, low cardinality and high cardinality strings, booleans).</li>
  <li>Hosting for the demo site is provided by Vercel.</li>
  </ul>
  <p dir="auto"><a href="https://vercel.com/?utm_source=datasette-visualization-plugin-demos&utm_campaign=oss" rel="nofollow"><img src="https://www.datocms-assets.com/31049/1618983297-powered-by-vercel.svg" alt="site hosted by vercel" style="max-width: 100%;" data-camo-src="https://camo.githubusercontent.com/275921faee4bc0e1a917948bc37c3b269daeb4ae4e37a51a62924d7c2867175d/68747470733a2f2f7777772e6461746f636d732d6173736574732e636f6d2f33313034392f313631383938333239372d706f77657265642d62792d76657263656c2e737667"></a></p>
  </article></div>


