1import asyncio 

2import asgi_csrf 

3import collections 

4import datetime 

5import hashlib 

6import itertools 

7import json 

8import os 

9import re 

10import secrets 

11import sys 

12import threading 

13import traceback 

14import urllib.parse 

15from concurrent import futures 

16from pathlib import Path 

17 

18import click 

19from markupsafe import Markup 

20from itsdangerous import URLSafeSerializer 

21import jinja2 

22from jinja2 import ChoiceLoader, Environment, FileSystemLoader, PrefixLoader, escape 

23from jinja2.environment import Template 

24from jinja2.exceptions import TemplateNotFound 

25import uvicorn 

26 

27from .views.base import DatasetteError, ureg, AsgiRouter 

28from .views.database import DatabaseDownload, DatabaseView 

29from .views.index import IndexView 

30from .views.special import ( 

31 JsonDataView, 

32 PatternPortfolioView, 

33 AuthTokenView, 

34 PermissionsDebugView, 

35 MessagesDebugView, 

36) 

37from .views.table import RowView, TableView 

38from .renderer import json_renderer 

39from .database import Database, QueryInterrupted 

40 

41from .utils import ( 

42 async_call_with_supported_arguments, 

43 escape_css_string, 

44 escape_sqlite, 

45 format_bytes, 

46 module_from_path, 

47 parse_metadata, 

48 resolve_env_secrets, 

49 sqlite3, 

50 to_css_class, 

51) 

52from .utils.asgi import ( 

53 AsgiLifespan, 

54 Forbidden, 

55 NotFound, 

56 Request, 

57 Response, 

58 asgi_static, 

59 asgi_send, 

60 asgi_send_html, 

61 asgi_send_json, 

62 asgi_send_redirect, 

63) 

64from .tracer import AsgiTracer 

65from .plugins import pm, DEFAULT_PLUGINS, get_plugins 

66from .version import __version__ 

67 

68app_root = Path(__file__).parent.parent 

69 

70MEMORY = object() 

71 

72ConfigOption = collections.namedtuple("ConfigOption", ("name", "default", "help")) 

73CONFIG_OPTIONS = ( 

74 ConfigOption("default_page_size", 100, "Default page size for the table view"), 

75 ConfigOption( 

76 "max_returned_rows", 

77 1000, 

78 "Maximum rows that can be returned from a table or custom query", 

79 ), 

80 ConfigOption( 

81 "num_sql_threads", 

82 3, 

83 "Number of threads in the thread pool for executing SQLite queries", 

84 ), 

85 ConfigOption( 

86 "sql_time_limit_ms", 1000, "Time limit for a SQL query in milliseconds" 

87 ), 

88 ConfigOption( 

89 "default_facet_size", 30, "Number of values to return for requested facets" 

90 ), 

91 ConfigOption( 

92 "facet_time_limit_ms", 200, "Time limit for calculating a requested facet" 

93 ), 

94 ConfigOption( 

95 "facet_suggest_time_limit_ms", 

96 50, 

97 "Time limit for calculating a suggested facet", 

98 ), 

99 ConfigOption( 

100 "hash_urls", 

101 False, 

102 "Include DB file contents hash in URLs, for far-future caching", 

103 ), 

104 ConfigOption( 

105 "allow_facet", 

106 True, 

107 "Allow users to specify columns to facet using ?_facet= parameter", 

108 ), 

109 ConfigOption( 

110 "allow_download", 

111 True, 

112 "Allow users to download the original SQLite database files", 

113 ), 

114 ConfigOption("suggest_facets", True, "Calculate and display suggested facets"), 

115 ConfigOption( 

116 "default_cache_ttl", 

117 5, 

118 "Default HTTP cache TTL (used in Cache-Control: max-age= header)", 

119 ), 

120 ConfigOption( 

121 "default_cache_ttl_hashed", 

122 365 * 24 * 60 * 60, 

123 "Default HTTP cache TTL for hashed URL pages", 

124 ), 

125 ConfigOption( 

126 "cache_size_kb", 0, "SQLite cache size in KB (0 == use SQLite default)" 

127 ), 

128 ConfigOption( 

129 "allow_csv_stream", 

130 True, 

131 "Allow .csv?_stream=1 to download all rows (ignoring max_returned_rows)", 

132 ), 

133 ConfigOption( 

134 "max_csv_mb", 

135 100, 

136 "Maximum size allowed for CSV export in MB - set 0 to disable this limit", 

137 ), 

138 ConfigOption( 

139 "truncate_cells_html", 

140 2048, 

141 "Truncate cells longer than this in HTML table view - set 0 to disable", 

142 ), 

143 ConfigOption( 

144 "force_https_urls", 

145 False, 

146 "Force URLs in API output to always use https:// protocol", 

147 ), 

148 ConfigOption( 

149 "template_debug", 

150 False, 

151 "Allow display of template debug information with ?_context=1", 

152 ), 

153 ConfigOption("base_url", "/", "Datasette URLs should use this base"), 

154) 

155 

156DEFAULT_CONFIG = {option.name: option.default for option in CONFIG_OPTIONS} 

157 

158 

159async def favicon(scope, receive, send): 

160 await asgi_send(send, "", 200) 

161 

162 

163class Datasette: 

164 # Message constants: 

165 INFO = 1 

166 WARNING = 2 

167 ERROR = 3 

168 

169 def __init__( 

170 self, 

171 files, 

172 immutables=None, 

173 cache_headers=True, 

174 cors=False, 

175 inspect_data=None, 

176 metadata=None, 

177 sqlite_extensions=None, 

178 template_dir=None, 

179 plugins_dir=None, 

180 static_mounts=None, 

181 memory=False, 

182 config=None, 

183 secret=None, 

184 version_note=None, 

185 config_dir=None, 

186 ): 

187 assert config_dir is None or isinstance( 

188 config_dir, Path 

189 ), "config_dir= should be a pathlib.Path" 

190 self._secret = secret or secrets.token_hex(32) 

191 self.files = tuple(files) + tuple(immutables or []) 

192 if config_dir: 

193 self.files += tuple([str(p) for p in config_dir.glob("*.db")]) 

194 if ( 

195 config_dir 

196 and (config_dir / "inspect-data.json").exists() 

197 and not inspect_data 

198 ): 

199 inspect_data = json.load((config_dir / "inspect-data.json").open()) 

200 if immutables is None: 

201 immutable_filenames = [i["file"] for i in inspect_data.values()] 

202 immutables = [ 

203 f for f in self.files if Path(f).name in immutable_filenames 

204 ] 

205 self.inspect_data = inspect_data 

206 self.immutables = set(immutables or []) 

207 if not self.files: 

208 self.files = [MEMORY] 

209 elif memory: 

210 self.files = (MEMORY,) + self.files 

211 self.databases = collections.OrderedDict() 

212 for file in self.files: 

213 path = file 

214 is_memory = False 

215 if file is MEMORY: 

216 path = None 

217 is_memory = True 

218 is_mutable = path not in self.immutables 

219 db = Database(self, path, is_mutable=is_mutable, is_memory=is_memory) 

220 if db.name in self.databases: 

221 raise Exception("Multiple files with same stem: {}".format(db.name)) 

222 self.add_database(db.name, db) 

223 self.cache_headers = cache_headers 

224 self.cors = cors 

225 metadata_files = [] 

226 if config_dir: 

227 metadata_files = [ 

228 config_dir / filename 

229 for filename in ("metadata.json", "metadata.yaml", "metadata.yml") 

230 if (config_dir / filename).exists() 

231 ] 

232 if config_dir and metadata_files and not metadata: 

233 with metadata_files[0].open() as fp: 

234 metadata = parse_metadata(fp.read()) 

235 self._metadata = metadata or {} 

236 self.sqlite_functions = [] 

237 self.sqlite_extensions = sqlite_extensions or [] 

238 if config_dir and (config_dir / "templates").is_dir() and not template_dir: 

239 template_dir = str((config_dir / "templates").resolve()) 

240 self.template_dir = template_dir 

241 if config_dir and (config_dir / "plugins").is_dir() and not plugins_dir: 

242 plugins_dir = str((config_dir / "plugins").resolve()) 

243 self.plugins_dir = plugins_dir 

244 if config_dir and (config_dir / "static").is_dir() and not static_mounts: 

245 static_mounts = [("static", str((config_dir / "static").resolve()))] 

246 self.static_mounts = static_mounts or [] 

247 if config_dir and (config_dir / "config.json").exists() and not config: 

248 config = json.load((config_dir / "config.json").open()) 

249 self._config = dict(DEFAULT_CONFIG, **(config or {})) 

250 self.renderers = {} # File extension -> (renderer, can_render) functions 

251 self.version_note = version_note 

252 self.executor = futures.ThreadPoolExecutor( 

253 max_workers=self.config("num_sql_threads") 

254 ) 

255 self.max_returned_rows = self.config("max_returned_rows") 

256 self.sql_time_limit_ms = self.config("sql_time_limit_ms") 

257 self.page_size = self.config("default_page_size") 

258 # Execute plugins in constructor, to ensure they are available 

259 # when the rest of `datasette inspect` executes 

260 if self.plugins_dir: 

261 for filename in os.listdir(self.plugins_dir): 

262 filepath = os.path.join(self.plugins_dir, filename) 

263 mod = module_from_path(filepath, name=filename) 

264 try: 

265 pm.register(mod) 

266 except ValueError: 

267 # Plugin already registered 

268 pass 

269 

270 # Configure Jinja 

271 default_templates = str(app_root / "datasette" / "templates") 

272 template_paths = [] 

273 if self.template_dir: 

274 template_paths.append(self.template_dir) 

275 plugin_template_paths = [ 

276 plugin["templates_path"] 

277 for plugin in get_plugins() 

278 if plugin["templates_path"] 

279 ] 

280 template_paths.extend(plugin_template_paths) 

281 template_paths.append(default_templates) 

282 template_loader = ChoiceLoader( 

283 [ 

284 FileSystemLoader(template_paths), 

285 # Support {% extends "default:table.html" %}: 

286 PrefixLoader( 

287 {"default": FileSystemLoader(default_templates)}, delimiter=":" 

288 ), 

289 ] 

290 ) 

291 self.jinja_env = Environment( 

292 loader=template_loader, autoescape=True, enable_async=True 

293 ) 

294 self.jinja_env.filters["escape_css_string"] = escape_css_string 

295 self.jinja_env.filters["quote_plus"] = lambda u: urllib.parse.quote_plus(u) 

296 self.jinja_env.filters["escape_sqlite"] = escape_sqlite 

297 self.jinja_env.filters["to_css_class"] = to_css_class 

298 # pylint: disable=no-member 

299 pm.hook.prepare_jinja2_environment(env=self.jinja_env) 

300 

301 self._register_renderers() 

302 self._permission_checks = collections.deque(maxlen=200) 

303 self._root_token = secrets.token_hex(32) 

304 

305 async def invoke_startup(self): 

306 for hook in pm.hook.startup(datasette=self): 

307 if callable(hook): 

308 hook = hook() 

309 if asyncio.iscoroutine(hook): 

310 hook = await hook 

311 

312 def sign(self, value, namespace="default"): 

313 return URLSafeSerializer(self._secret, namespace).dumps(value) 

314 

315 def unsign(self, signed, namespace="default"): 

316 return URLSafeSerializer(self._secret, namespace).loads(signed) 

317 

318 def get_database(self, name=None): 

319 if name is None: 

320 return next(iter(self.databases.values())) 

321 return self.databases[name] 

322 

323 def add_database(self, name, db): 

324 self.databases[name] = db 

325 

326 def remove_database(self, name): 

327 self.databases.pop(name) 

328 

329 def config(self, key): 

330 return self._config.get(key, None) 

331 

332 def config_dict(self): 

333 # Returns a fully resolved config dictionary, useful for templates 

334 return {option.name: self.config(option.name) for option in CONFIG_OPTIONS} 

335 

336 def metadata(self, key=None, database=None, table=None, fallback=True): 

337 """ 

338 Looks up metadata, cascading backwards from specified level. 

339 Returns None if metadata value is not found. 

340 """ 

341 assert not ( 

342 database is None and table is not None 

343 ), "Cannot call metadata() with table= specified but not database=" 

344 databases = self._metadata.get("databases") or {} 

345 search_list = [] 

346 if database is not None: 

347 search_list.append(databases.get(database) or {}) 

348 if table is not None: 

349 table_metadata = ((databases.get(database) or {}).get("tables") or {}).get( 

350 table 

351 ) or {} 

352 search_list.insert(0, table_metadata) 

353 search_list.append(self._metadata) 

354 if not fallback: 

355 # No fallback allowed, so just use the first one in the list 

356 search_list = search_list[:1] 

357 if key is not None: 

358 for item in search_list: 

359 if key in item: 

360 return item[key] 

361 return None 

362 else: 

363 # Return the merged list 

364 m = {} 

365 for item in search_list: 

366 m.update(item) 

367 return m 

368 

369 def plugin_config(self, plugin_name, database=None, table=None, fallback=True): 

370 "Return config for plugin, falling back from specified database/table" 

371 plugins = self.metadata( 

372 "plugins", database=database, table=table, fallback=fallback 

373 ) 

374 if plugins is None: 

375 return None 

376 plugin_config = plugins.get(plugin_name) 

377 # Resolve any $file and $env keys 

378 plugin_config = resolve_env_secrets(plugin_config, os.environ) 

379 return plugin_config 

380 

381 def app_css_hash(self): 

382 if not hasattr(self, "_app_css_hash"): 

383 self._app_css_hash = hashlib.sha1( 

384 open(os.path.join(str(app_root), "datasette/static/app.css")) 

385 .read() 

386 .encode("utf8") 

387 ).hexdigest()[:6] 

388 return self._app_css_hash 

389 

390 def get_canned_queries(self, database_name): 

391 queries = self.metadata("queries", database=database_name, fallback=False) or {} 

392 names = queries.keys() 

393 return [self.get_canned_query(database_name, name) for name in names] 

394 

395 def get_canned_query(self, database_name, query_name): 

396 queries = self.metadata("queries", database=database_name, fallback=False) or {} 

397 query = queries.get(query_name) 

398 if query: 

399 if not isinstance(query, dict): 

400 query = {"sql": query} 

401 query["name"] = query_name 

402 return query 

403 

404 def update_with_inherited_metadata(self, metadata): 

405 # Fills in source/license with defaults, if available 

406 metadata.update( 

407 { 

408 "source": metadata.get("source") or self.metadata("source"), 

409 "source_url": metadata.get("source_url") or self.metadata("source_url"), 

410 "license": metadata.get("license") or self.metadata("license"), 

411 "license_url": metadata.get("license_url") 

412 or self.metadata("license_url"), 

413 "about": metadata.get("about") or self.metadata("about"), 

414 "about_url": metadata.get("about_url") or self.metadata("about_url"), 

415 } 

416 ) 

417 

418 def _prepare_connection(self, conn, database): 

419 conn.row_factory = sqlite3.Row 

420 conn.text_factory = lambda x: str(x, "utf-8", "replace") 

421 for name, num_args, func in self.sqlite_functions: 

422 conn.create_function(name, num_args, func) 

423 if self.sqlite_extensions: 

424 conn.enable_load_extension(True) 

425 for extension in self.sqlite_extensions: 

426 conn.execute("SELECT load_extension('{}')".format(extension)) 

427 if self.config("cache_size_kb"): 

428 conn.execute("PRAGMA cache_size=-{}".format(self.config("cache_size_kb"))) 

429 # pylint: disable=no-member 

430 pm.hook.prepare_connection(conn=conn, database=database, datasette=self) 

431 

432 def add_message(self, request, message, type=INFO): 

433 if not hasattr(request, "_messages"): 

434 request._messages = [] 

435 request._messages_should_clear = False 

436 request._messages.append((message, type)) 

437 

438 def _write_messages_to_response(self, request, response): 

439 if getattr(request, "_messages", None): 

440 # Set those messages 

441 response.set_cookie("ds_messages", self.sign(request._messages, "messages")) 

442 elif getattr(request, "_messages_should_clear", False): 

443 response.set_cookie("ds_messages", "", expires=0, max_age=0) 

444 

445 def _show_messages(self, request): 

446 if getattr(request, "_messages", None): 

447 request._messages_should_clear = True 

448 messages = request._messages 

449 request._messages = [] 

450 return messages 

451 else: 

452 return [] 

453 

454 async def permission_allowed(self, actor, action, resource=None, default=False): 

455 "Check permissions using the permissions_allowed plugin hook" 

456 result = None 

457 for check in pm.hook.permission_allowed( 

458 datasette=self, actor=actor, action=action, resource=resource, 

459 ): 

460 if callable(check): 

461 check = check() 

462 if asyncio.iscoroutine(check): 

463 check = await check 

464 if check is not None: 

465 result = check 

466 used_default = False 

467 if result is None: 

468 result = default 

469 used_default = True 

470 self._permission_checks.append( 

471 { 

472 "when": datetime.datetime.utcnow().isoformat(), 

473 "actor": actor, 

474 "action": action, 

475 "resource": resource, 

476 "used_default": used_default, 

477 "result": result, 

478 } 

479 ) 

480 return result 

481 

482 async def execute( 

483 self, 

484 db_name, 

485 sql, 

486 params=None, 

487 truncate=False, 

488 custom_time_limit=None, 

489 page_size=None, 

490 log_sql_errors=True, 

491 ): 

492 return await self.databases[db_name].execute( 

493 sql, 

494 params=params, 

495 truncate=truncate, 

496 custom_time_limit=custom_time_limit, 

497 page_size=page_size, 

498 log_sql_errors=log_sql_errors, 

499 ) 

500 

501 async def expand_foreign_keys(self, database, table, column, values): 

502 "Returns dict mapping (column, value) -> label" 

503 labeled_fks = {} 

504 db = self.databases[database] 

505 foreign_keys = await db.foreign_keys_for_table(table) 

506 # Find the foreign_key for this column 

507 try: 

508 fk = [ 

509 foreign_key 

510 for foreign_key in foreign_keys 

511 if foreign_key["column"] == column 

512 ][0] 

513 except IndexError: 

514 return {} 

515 label_column = await db.label_column_for_table(fk["other_table"]) 

516 if not label_column: 

517 return {(fk["column"], value): str(value) for value in values} 

518 labeled_fks = {} 

519 sql = """ 

520 select {other_column}, {label_column} 

521 from {other_table} 

522 where {other_column} in ({placeholders}) 

523 """.format( 

524 other_column=escape_sqlite(fk["other_column"]), 

525 label_column=escape_sqlite(label_column), 

526 other_table=escape_sqlite(fk["other_table"]), 

527 placeholders=", ".join(["?"] * len(set(values))), 

528 ) 

529 try: 

530 results = await self.execute(database, sql, list(set(values))) 

531 except QueryInterrupted: 

532 pass 

533 else: 

534 for id, value in results: 

535 labeled_fks[(fk["column"], id)] = value 

536 return labeled_fks 

537 

538 def absolute_url(self, request, path): 

539 url = urllib.parse.urljoin(request.url, path) 

540 if url.startswith("http://") and self.config("force_https_urls"): 

541 url = "https://" + url[len("http://") :] 

542 return url 

543 

544 def _register_custom_units(self): 

545 "Register any custom units defined in the metadata.json with Pint" 

546 for unit in self.metadata("custom_units") or []: 

547 ureg.define(unit) 

548 

549 def _connected_databases(self): 

550 return [ 

551 { 

552 "name": d.name, 

553 "path": d.path, 

554 "size": d.size, 

555 "is_mutable": d.is_mutable, 

556 "is_memory": d.is_memory, 

557 "hash": d.hash, 

558 } 

559 for d in sorted(self.databases.values(), key=lambda d: d.name) 

560 ] 

561 

562 def _versions(self): 

563 conn = sqlite3.connect(":memory:") 

564 self._prepare_connection(conn, ":memory:") 

565 sqlite_version = conn.execute("select sqlite_version()").fetchone()[0] 

566 sqlite_extensions = {} 

567 for extension, testsql, hasversion in ( 

568 ("json1", "SELECT json('{}')", False), 

569 ("spatialite", "SELECT spatialite_version()", True), 

570 ): 

571 try: 

572 result = conn.execute(testsql) 

573 if hasversion: 

574 sqlite_extensions[extension] = result.fetchone()[0] 

575 else: 

576 sqlite_extensions[extension] = None 

577 except Exception: 

578 pass 

579 # Figure out supported FTS versions 

580 fts_versions = [] 

581 for fts in ("FTS5", "FTS4", "FTS3"): 

582 try: 

583 conn.execute( 

584 "CREATE VIRTUAL TABLE v{fts} USING {fts} (data)".format(fts=fts) 

585 ) 

586 fts_versions.append(fts) 

587 except sqlite3.OperationalError: 

588 continue 

589 datasette_version = {"version": __version__} 

590 if self.version_note: 

591 datasette_version["note"] = self.version_note 

592 return { 

593 "python": { 

594 "version": ".".join(map(str, sys.version_info[:3])), 

595 "full": sys.version, 

596 }, 

597 "datasette": datasette_version, 

598 "asgi": "3.0", 

599 "uvicorn": uvicorn.__version__, 

600 "sqlite": { 

601 "version": sqlite_version, 

602 "fts_versions": fts_versions, 

603 "extensions": sqlite_extensions, 

604 "compile_options": [ 

605 r[0] for r in conn.execute("pragma compile_options;").fetchall() 

606 ], 

607 }, 

608 } 

609 

610 def _plugins(self, request=None, all=False): 

611 ps = list(get_plugins()) 

612 should_show_all = False 

613 if request is not None: 

614 should_show_all = request.args.get("all") 

615 else: 

616 should_show_all = all 

617 if not should_show_all: 

618 ps = [p for p in ps if p["name"] not in DEFAULT_PLUGINS] 

619 return [ 

620 { 

621 "name": p["name"], 

622 "static": p["static_path"] is not None, 

623 "templates": p["templates_path"] is not None, 

624 "version": p.get("version"), 

625 "hooks": p["hooks"], 

626 } 

627 for p in ps 

628 ] 

629 

630 def _threads(self): 

631 threads = list(threading.enumerate()) 

632 d = { 

633 "num_threads": len(threads), 

634 "threads": [ 

635 {"name": t.name, "ident": t.ident, "daemon": t.daemon} for t in threads 

636 ], 

637 } 

638 # Only available in Python 3.7+ 

639 if hasattr(asyncio, "all_tasks"): 

640 tasks = asyncio.all_tasks() 

641 d.update( 

642 { 

643 "num_tasks": len(tasks), 

644 "tasks": [_cleaner_task_str(t) for t in tasks], 

645 } 

646 ) 

647 return d 

648 

649 def _actor(self, request): 

650 return {"actor": request.actor} 

651 

652 def table_metadata(self, database, table): 

653 "Fetch table-specific metadata." 

654 return ( 

655 (self.metadata("databases") or {}) 

656 .get(database, {}) 

657 .get("tables", {}) 

658 .get(table, {}) 

659 ) 

660 

661 def _register_renderers(self): 

662 """ Register output renderers which output data in custom formats. """ 

663 # Built-in renderers 

664 self.renderers["json"] = (json_renderer, lambda: True) 

665 

666 # Hooks 

667 hook_renderers = [] 

668 # pylint: disable=no-member 

669 for hook in pm.hook.register_output_renderer(datasette=self): 

670 if type(hook) == list: 

671 hook_renderers += hook 

672 else: 

673 hook_renderers.append(hook) 

674 

675 for renderer in hook_renderers: 

676 self.renderers[renderer["extension"]] = ( 

677 # It used to be called "callback" - remove this in Datasette 1.0 

678 renderer.get("render") or renderer["callback"], 

679 renderer.get("can_render") or (lambda: True), 

680 ) 

681 

682 async def render_template( 

683 self, templates, context=None, request=None, view_name=None 

684 ): 

685 context = context or {} 

686 if isinstance(templates, Template): 

687 template = templates 

688 else: 

689 if isinstance(templates, str): 

690 templates = [templates] 

691 template = self.jinja_env.select_template(templates) 

692 body_scripts = [] 

693 # pylint: disable=no-member 

694 for script in pm.hook.extra_body_script( 

695 template=template.name, 

696 database=context.get("database"), 

697 table=context.get("table"), 

698 view_name=view_name, 

699 datasette=self, 

700 ): 

701 body_scripts.append(Markup(script)) 

702 

703 extra_template_vars = {} 

704 # pylint: disable=no-member 

705 for extra_vars in pm.hook.extra_template_vars( 

706 template=template.name, 

707 database=context.get("database"), 

708 table=context.get("table"), 

709 view_name=view_name, 

710 request=request, 

711 datasette=self, 

712 ): 

713 if callable(extra_vars): 

714 extra_vars = extra_vars() 

715 if asyncio.iscoroutine(extra_vars): 

716 extra_vars = await extra_vars 

717 assert isinstance(extra_vars, dict), "extra_vars is of type {}".format( 

718 type(extra_vars) 

719 ) 

720 extra_template_vars.update(extra_vars) 

721 

722 template_context = { 

723 **context, 

724 **{ 

725 "app_css_hash": self.app_css_hash(), 

726 "zip": zip, 

727 "body_scripts": body_scripts, 

728 "format_bytes": format_bytes, 

729 "extra_css_urls": self._asset_urls("extra_css_urls", template, context), 

730 "extra_js_urls": self._asset_urls("extra_js_urls", template, context), 

731 "base_url": self.config("base_url"), 

732 }, 

733 **extra_template_vars, 

734 } 

735 if request and request.args.get("_context") and self.config("template_debug"): 

736 return "<pre>{}</pre>".format( 

737 jinja2.escape(json.dumps(template_context, default=repr, indent=4)) 

738 ) 

739 

740 return await template.render_async(template_context) 

741 

742 def _asset_urls(self, key, template, context): 

743 # Flatten list-of-lists from plugins: 

744 seen_urls = set() 

745 for url_or_dict in itertools.chain( 

746 itertools.chain.from_iterable( 

747 getattr(pm.hook, key)( 

748 template=template.name, 

749 database=context.get("database"), 

750 table=context.get("table"), 

751 datasette=self, 

752 ) 

753 ), 

754 (self.metadata(key) or []), 

755 ): 

756 if isinstance(url_or_dict, dict): 

757 url = url_or_dict["url"] 

758 sri = url_or_dict.get("sri") 

759 else: 

760 url = url_or_dict 

761 sri = None 

762 if url in seen_urls: 

763 continue 

764 seen_urls.add(url) 

765 if sri: 

766 yield {"url": url, "sri": sri} 

767 else: 

768 yield {"url": url} 

769 

770 def app(self): 

771 "Returns an ASGI app function that serves the whole of Datasette" 

772 routes = [] 

773 

774 for routes_to_add in pm.hook.register_routes(): 

775 for regex, view_fn in routes_to_add: 

776 routes.append((regex, wrap_view(view_fn, self))) 

777 

778 def add_route(view, regex): 

779 routes.append((regex, view)) 

780 

781 # Generate a regex snippet to match all registered renderer file extensions 

782 renderer_regex = "|".join(r"\." + key for key in self.renderers.keys()) 

783 

784 add_route(IndexView.as_asgi(self), r"/(?P<as_format>(\.jsono?)?$)") 

785 # TODO: /favicon.ico and /-/static/ deserve far-future cache expires 

786 add_route(favicon, "/favicon.ico") 

787 

788 add_route( 

789 asgi_static(app_root / "datasette" / "static"), r"/-/static/(?P<path>.*)$" 

790 ) 

791 for path, dirname in self.static_mounts: 

792 add_route(asgi_static(dirname), r"/" + path + "/(?P<path>.*)$") 

793 

794 # Mount any plugin static/ directories 

795 for plugin in get_plugins(): 

796 if plugin["static_path"]: 

797 add_route( 

798 asgi_static(plugin["static_path"]), 

799 "/-/static-plugins/{}/(?P<path>.*)$".format(plugin["name"]), 

800 ) 

801 # Support underscores in name in addition to hyphens, see https://github.com/simonw/datasette/issues/611 

802 add_route( 

803 asgi_static(plugin["static_path"]), 

804 "/-/static-plugins/{}/(?P<path>.*)$".format( 

805 plugin["name"].replace("-", "_") 

806 ), 

807 ) 

808 add_route( 

809 JsonDataView.as_asgi(self, "metadata.json", lambda: self._metadata), 

810 r"/-/metadata(?P<as_format>(\.json)?)$", 

811 ) 

812 add_route( 

813 JsonDataView.as_asgi(self, "versions.json", self._versions), 

814 r"/-/versions(?P<as_format>(\.json)?)$", 

815 ) 

816 add_route( 

817 JsonDataView.as_asgi( 

818 self, "plugins.json", self._plugins, needs_request=True 

819 ), 

820 r"/-/plugins(?P<as_format>(\.json)?)$", 

821 ) 

822 add_route( 

823 JsonDataView.as_asgi(self, "config.json", lambda: self._config), 

824 r"/-/config(?P<as_format>(\.json)?)$", 

825 ) 

826 add_route( 

827 JsonDataView.as_asgi(self, "threads.json", self._threads), 

828 r"/-/threads(?P<as_format>(\.json)?)$", 

829 ) 

830 add_route( 

831 JsonDataView.as_asgi(self, "databases.json", self._connected_databases), 

832 r"/-/databases(?P<as_format>(\.json)?)$", 

833 ) 

834 add_route( 

835 JsonDataView.as_asgi(self, "actor.json", self._actor, needs_request=True), 

836 r"/-/actor(?P<as_format>(\.json)?)$", 

837 ) 

838 add_route( 

839 AuthTokenView.as_asgi(self), r"/-/auth-token$", 

840 ) 

841 add_route( 

842 PermissionsDebugView.as_asgi(self), r"/-/permissions$", 

843 ) 

844 add_route( 

845 MessagesDebugView.as_asgi(self), r"/-/messages$", 

846 ) 

847 add_route( 

848 PatternPortfolioView.as_asgi(self), r"/-/patterns$", 

849 ) 

850 add_route( 

851 DatabaseDownload.as_asgi(self), r"/(?P<db_name>[^/]+?)(?P<as_db>\.db)$" 

852 ) 

853 add_route( 

854 DatabaseView.as_asgi(self), 

855 r"/(?P<db_name>[^/]+?)(?P<as_format>" 

856 + renderer_regex 

857 + r"|.jsono|\.csv)?$", 

858 ) 

859 add_route( 

860 TableView.as_asgi(self), 

861 r"/(?P<db_name>[^/]+)/(?P<table_and_format>[^/]+?$)", 

862 ) 

863 add_route( 

864 RowView.as_asgi(self), 

865 r"/(?P<db_name>[^/]+)/(?P<table>[^/]+?)/(?P<pk_path>[^/]+?)(?P<as_format>" 

866 + renderer_regex 

867 + r")?$", 

868 ) 

869 self._register_custom_units() 

870 

871 async def setup_db(): 

872 # First time server starts up, calculate table counts for immutable databases 

873 for dbname, database in self.databases.items(): 

874 if not database.is_mutable: 

875 await database.table_counts(limit=60 * 60 * 1000) 

876 

877 asgi = AsgiLifespan( 

878 AsgiTracer( 

879 asgi_csrf.asgi_csrf( 

880 DatasetteRouter(self, routes), 

881 signing_secret=self._secret, 

882 cookie_name="ds_csrftoken", 

883 ) 

884 ), 

885 on_startup=setup_db, 

886 ) 

887 for wrapper in pm.hook.asgi_wrapper(datasette=self): 

888 asgi = wrapper(asgi) 

889 return asgi 

890 

891 

892class DatasetteRouter(AsgiRouter): 

893 def __init__(self, datasette, routes): 

894 self.ds = datasette 

895 super().__init__(routes) 

896 

897 async def route_path(self, scope, receive, send, path): 

898 # Strip off base_url if present before routing 

899 base_url = self.ds.config("base_url") 

900 if base_url != "/" and path.startswith(base_url): 

901 path = "/" + path[len(base_url) :] 

902 scope_modifications = {} 

903 # Apply force_https_urls, if set 

904 if ( 

905 self.ds.config("force_https_urls") 

906 and scope["type"] == "http" 

907 and scope.get("scheme") != "https" 

908 ): 

909 scope_modifications["scheme"] = "https" 

910 # Handle authentication 

911 actor = None 

912 for actor in pm.hook.actor_from_request( 

913 datasette=self.ds, request=Request(scope, receive) 

914 ): 

915 if callable(actor): 

916 actor = actor() 

917 if asyncio.iscoroutine(actor): 

918 actor = await actor 

919 if actor: 

920 break 

921 scope_modifications["actor"] = actor 

922 return await super().route_path( 

923 dict(scope, **scope_modifications), receive, send, path 

924 ) 

925 

926 async def handle_404(self, scope, receive, send, exception=None): 

927 # If URL has a trailing slash, redirect to URL without it 

928 path = scope.get("raw_path", scope["path"].encode("utf8")) 

929 if path.endswith(b"/"): 

930 path = path.rstrip(b"/") 

931 if scope["query_string"]: 

932 path += b"?" + scope["query_string"] 

933 await asgi_send_redirect(send, path.decode("latin1")) 

934 else: 

935 # Is there a pages/* template matching this path? 

936 template_path = os.path.join("pages", *scope["path"].split("/")) + ".html" 

937 try: 

938 template = self.ds.jinja_env.select_template([template_path]) 

939 except TemplateNotFound: 

940 template = None 

941 if template: 

942 headers = {} 

943 status = [200] 

944 

945 def custom_header(name, value): 

946 headers[name] = value 

947 return "" 

948 

949 def custom_status(code): 

950 status[0] = code 

951 return "" 

952 

953 def custom_redirect(location, code=302): 

954 status[0] = code 

955 headers["Location"] = location 

956 return "" 

957 

958 body = await self.ds.render_template( 

959 template, 

960 { 

961 "custom_header": custom_header, 

962 "custom_status": custom_status, 

963 "custom_redirect": custom_redirect, 

964 }, 

965 request=Request(scope, receive), 

966 view_name="page", 

967 ) 

968 # Pull content-type out into separate parameter 

969 content_type = "text/html; charset=utf-8" 

970 matches = [k for k in headers if k.lower() == "content-type"] 

971 if matches: 

972 content_type = headers[matches[0]] 

973 await asgi_send( 

974 send, 

975 body, 

976 status=status[0], 

977 headers=headers, 

978 content_type=content_type, 

979 ) 

980 else: 

981 await self.handle_500( 

982 scope, receive, send, exception or NotFound("404") 

983 ) 

984 

985 async def handle_500(self, scope, receive, send, exception): 

986 title = None 

987 if isinstance(exception, NotFound): 

988 status = 404 

989 info = {} 

990 message = exception.args[0] 

991 elif isinstance(exception, Forbidden): 

992 status = 403 

993 info = {} 

994 message = exception.args[0] 

995 elif isinstance(exception, DatasetteError): 

996 status = exception.status 

997 info = exception.error_dict 

998 message = exception.message 

999 if exception.messagge_is_html: 

1000 message = Markup(message) 

1001 title = exception.title 

1002 else: 

1003 status = 500 

1004 info = {} 

1005 message = str(exception) 

1006 traceback.print_exc() 

1007 templates = ["500.html"] 

1008 if status != 500: 

1009 templates = ["{}.html".format(status)] + templates 

1010 info.update( 

1011 {"ok": False, "error": message, "status": status, "title": title,} 

1012 ) 

1013 headers = {} 

1014 if self.ds.cors: 

1015 headers["Access-Control-Allow-Origin"] = "*" 

1016 if scope["path"].split("?")[0].endswith(".json"): 

1017 await asgi_send_json(send, info, status=status, headers=headers) 

1018 else: 

1019 template = self.ds.jinja_env.select_template(templates) 

1020 await asgi_send_html( 

1021 send, 

1022 await template.render_async( 

1023 dict( 

1024 info, 

1025 base_url=self.ds.config("base_url"), 

1026 app_css_hash=self.ds.app_css_hash(), 

1027 ) 

1028 ), 

1029 status=status, 

1030 headers=headers, 

1031 ) 

1032 

1033 

1034_cleaner_task_str_re = re.compile(r"\S*site-packages/") 

1035 

1036 

1037def _cleaner_task_str(task): 

1038 s = str(task) 

1039 # This has something like the following in it: 

1040 # running at /Users/simonw/Dropbox/Development/datasette/venv-3.7.5/lib/python3.7/site-packages/uvicorn/main.py:361> 

1041 # Clean up everything up to and including site-packages 

1042 return _cleaner_task_str_re.sub("", s) 

1043 

1044 

1045def wrap_view(view_fn, datasette): 

1046 async def asgi_view_fn(scope, receive, send): 

1047 response = await async_call_with_supported_arguments( 

1048 view_fn, 

1049 scope=scope, 

1050 receive=receive, 

1051 send=send, 

1052 request=Request(scope, receive), 

1053 datasette=datasette, 

1054 ) 

1055 if response is not None: 

1056 await response.asgi_send(send) 

1057 

1058 return asgi_view_fn