1from contextlib import contextmanager 

2from collections import OrderedDict 

3import base64 

4import click 

5import hashlib 

6import inspect 

7import json 

8import mergedeep 

9import os 

10import re 

11import shlex 

12import tempfile 

13import time 

14import types 

15import shutil 

16import urllib 

17import numbers 

18import yaml 

19from .shutil_backport import copytree 

20 

21try: 

22 import pysqlite3 as sqlite3 

23except ImportError: 

24 import sqlite3 

25 

26# From https://www.sqlite.org/lang_keywords.html 

27reserved_words = set( 

28 ( 

29 "abort action add after all alter analyze and as asc attach autoincrement " 

30 "before begin between by cascade case cast check collate column commit " 

31 "conflict constraint create cross current_date current_time " 

32 "current_timestamp database default deferrable deferred delete desc detach " 

33 "distinct drop each else end escape except exclusive exists explain fail " 

34 "for foreign from full glob group having if ignore immediate in index " 

35 "indexed initially inner insert instead intersect into is isnull join key " 

36 "left like limit match natural no not notnull null of offset on or order " 

37 "outer plan pragma primary query raise recursive references regexp reindex " 

38 "release rename replace restrict right rollback row savepoint select set " 

39 "table temp temporary then to transaction trigger union unique update using " 

40 "vacuum values view virtual when where with without" 

41 ).split() 

42) 

43 

44SPATIALITE_DOCKERFILE_EXTRAS = r""" 

45RUN apt-get update && \ 

46 apt-get install -y python3-dev gcc libsqlite3-mod-spatialite && \ 

47 rm -rf /var/lib/apt/lists/* 

48ENV SQLITE_EXTENSIONS /usr/lib/x86_64-linux-gnu/mod_spatialite.so 

49""" 

50 

51 

52def urlsafe_components(token): 

53 "Splits token on commas and URL decodes each component" 

54 return [urllib.parse.unquote_plus(b) for b in token.split(",")] 

55 

56 

57def path_from_row_pks(row, pks, use_rowid, quote=True): 

58 """ Generate an optionally URL-quoted unique identifier 

59 for a row from its primary keys.""" 

60 if use_rowid: 

61 bits = [row["rowid"]] 

62 else: 

63 bits = [ 

64 row[pk]["value"] if isinstance(row[pk], dict) else row[pk] for pk in pks 

65 ] 

66 if quote: 

67 bits = [urllib.parse.quote_plus(str(bit)) for bit in bits] 

68 else: 

69 bits = [str(bit) for bit in bits] 

70 

71 return ",".join(bits) 

72 

73 

74def compound_keys_after_sql(pks, start_index=0): 

75 # Implementation of keyset pagination 

76 # See https://github.com/simonw/datasette/issues/190 

77 # For pk1/pk2/pk3 returns: 

78 # 

79 # ([pk1] > :p0) 

80 # or 

81 # ([pk1] = :p0 and [pk2] > :p1) 

82 # or 

83 # ([pk1] = :p0 and [pk2] = :p1 and [pk3] > :p2) 

84 or_clauses = [] 

85 pks_left = pks[:] 

86 while pks_left: 

87 and_clauses = [] 

88 last = pks_left[-1] 

89 rest = pks_left[:-1] 

90 and_clauses = [ 

91 "{} = :p{}".format(escape_sqlite(pk), (i + start_index)) 

92 for i, pk in enumerate(rest) 

93 ] 

94 and_clauses.append( 

95 "{} > :p{}".format(escape_sqlite(last), (len(rest) + start_index)) 

96 ) 

97 or_clauses.append("({})".format(" and ".join(and_clauses))) 

98 pks_left.pop() 

99 or_clauses.reverse() 

100 return "({})".format("\n or\n".join(or_clauses)) 

101 

102 

103class CustomJSONEncoder(json.JSONEncoder): 

104 def default(self, obj): 

105 if isinstance(obj, sqlite3.Row): 

106 return tuple(obj) 

107 if isinstance(obj, sqlite3.Cursor): 

108 return list(obj) 

109 if isinstance(obj, bytes): 

110 # Does it encode to utf8? 

111 try: 

112 return obj.decode("utf8") 

113 except UnicodeDecodeError: 

114 return { 

115 "$base64": True, 

116 "encoded": base64.b64encode(obj).decode("latin1"), 

117 } 

118 return json.JSONEncoder.default(self, obj) 

119 

120 

121@contextmanager 

122def sqlite_timelimit(conn, ms): 

123 deadline = time.time() + (ms / 1000) 

124 # n is the number of SQLite virtual machine instructions that will be 

125 # executed between each check. It's hard to know what to pick here. 

126 # After some experimentation, I've decided to go with 1000 by default and 

127 # 1 for time limits that are less than 50ms 

128 n = 1000 

129 if ms < 50: 

130 n = 1 

131 

132 def handler(): 

133 if time.time() >= deadline: 

134 return 1 

135 

136 conn.set_progress_handler(handler, n) 

137 try: 

138 yield 

139 finally: 

140 conn.set_progress_handler(None, n) 

141 

142 

143class InvalidSql(Exception): 

144 pass 

145 

146 

147allowed_sql_res = [ 

148 re.compile(r"^select\b"), 

149 re.compile(r"^explain select\b"), 

150 re.compile(r"^explain query plan select\b"), 

151 re.compile(r"^with\b"), 

152 re.compile(r"^explain with\b"), 

153 re.compile(r"^explain query plan with\b"), 

154] 

155allowed_pragmas = ( 

156 "database_list", 

157 "foreign_key_list", 

158 "function_list", 

159 "index_info", 

160 "index_list", 

161 "index_xinfo", 

162 "page_count", 

163 "max_page_count", 

164 "page_size", 

165 "schema_version", 

166 "table_info", 

167 "table_xinfo", 

168) 

169disallawed_sql_res = [ 

170 ( 

171 re.compile("pragma(?!_({}))".format("|".join(allowed_pragmas))), 

172 "Statement may not contain PRAGMA", 

173 ) 

174] 

175 

176 

177def validate_sql_select(sql): 

178 sql = "\n".join( 

179 line for line in sql.split("\n") if not line.strip().startswith("--") 

180 ) 

181 sql = sql.strip().lower() 

182 if not any(r.match(sql) for r in allowed_sql_res): 

183 raise InvalidSql("Statement must be a SELECT") 

184 for r, msg in disallawed_sql_res: 

185 if r.search(sql): 

186 raise InvalidSql(msg) 

187 

188 

189def append_querystring(url, querystring): 

190 op = "&" if ("?" in url) else "?" 

191 return "{}{}{}".format(url, op, querystring) 

192 

193 

194def path_with_added_args(request, args, path=None): 

195 path = path or request.path 

196 if isinstance(args, dict): 

197 args = args.items() 

198 args_to_remove = {k for k, v in args if v is None} 

199 current = [] 

200 for key, value in urllib.parse.parse_qsl(request.query_string): 

201 if key not in args_to_remove: 

202 current.append((key, value)) 

203 current.extend([(key, value) for key, value in args if value is not None]) 

204 query_string = urllib.parse.urlencode(current) 

205 if query_string: 

206 query_string = "?{}".format(query_string) 

207 return path + query_string 

208 

209 

210def path_with_removed_args(request, args, path=None): 

211 query_string = request.query_string 

212 if path is None: 

213 path = request.path 

214 else: 

215 if "?" in path: 

216 bits = path.split("?", 1) 

217 path, query_string = bits 

218 # args can be a dict or a set 

219 current = [] 

220 if isinstance(args, set): 

221 

222 def should_remove(key, value): 

223 return key in args 

224 

225 elif isinstance(args, dict): 

226 # Must match key AND value 

227 def should_remove(key, value): 

228 return args.get(key) == value 

229 

230 for key, value in urllib.parse.parse_qsl(query_string): 

231 if not should_remove(key, value): 

232 current.append((key, value)) 

233 query_string = urllib.parse.urlencode(current) 

234 if query_string: 

235 query_string = "?{}".format(query_string) 

236 return path + query_string 

237 

238 

239def path_with_replaced_args(request, args, path=None): 

240 path = path or request.path 

241 if isinstance(args, dict): 

242 args = args.items() 

243 keys_to_replace = {p[0] for p in args} 

244 current = [] 

245 for key, value in urllib.parse.parse_qsl(request.query_string): 

246 if key not in keys_to_replace: 

247 current.append((key, value)) 

248 current.extend([p for p in args if p[1] is not None]) 

249 query_string = urllib.parse.urlencode(current) 

250 if query_string: 

251 query_string = "?{}".format(query_string) 

252 return path + query_string 

253 

254 

255_css_re = re.compile(r"""['"\n\\]""") 

256_boring_keyword_re = re.compile(r"^[a-zA-Z_][a-zA-Z0-9_]*$") 

257 

258 

259def escape_css_string(s): 

260 return _css_re.sub(lambda m: "\\{:X}".format(ord(m.group())), s) 

261 

262 

263def escape_sqlite(s): 

264 if _boring_keyword_re.match(s) and (s.lower() not in reserved_words): 

265 return s 

266 else: 

267 return "[{}]".format(s) 

268 

269 

270def make_dockerfile( 

271 files, 

272 metadata_file, 

273 extra_options, 

274 branch, 

275 template_dir, 

276 plugins_dir, 

277 static, 

278 install, 

279 spatialite, 

280 version_note, 

281 secret, 

282 environment_variables=None, 

283 port=8001, 

284): 

285 cmd = ["datasette", "serve", "--host", "0.0.0.0"] 

286 environment_variables = environment_variables or {} 

287 environment_variables["DATASETTE_SECRET"] = secret 

288 for filename in files: 

289 cmd.extend(["-i", filename]) 

290 cmd.extend(["--cors", "--inspect-file", "inspect-data.json"]) 

291 if metadata_file: 

292 cmd.extend(["--metadata", "{}".format(metadata_file)]) 

293 if template_dir: 

294 cmd.extend(["--template-dir", "templates/"]) 

295 if plugins_dir: 

296 cmd.extend(["--plugins-dir", "plugins/"]) 

297 if version_note: 

298 cmd.extend(["--version-note", "{}".format(version_note)]) 

299 if static: 

300 for mount_point, _ in static: 

301 cmd.extend(["--static", "{}:{}".format(mount_point, mount_point)]) 

302 if extra_options: 

303 for opt in extra_options.split(): 

304 cmd.append("{}".format(opt)) 

305 cmd = [shlex.quote(part) for part in cmd] 

306 # port attribute is a (fixed) env variable and should not be quoted 

307 cmd.extend(["--port", "$PORT"]) 

308 cmd = " ".join(cmd) 

309 if branch: 

310 install = [ 

311 "https://github.com/simonw/datasette/archive/{}.zip".format(branch) 

312 ] + list(install) 

313 else: 

314 install = ["datasette"] + list(install) 

315 

316 return """ 

317FROM python:3.8 

318COPY . /app 

319WORKDIR /app 

320{spatialite_extras} 

321{environment_variables} 

322RUN pip install -U {install_from} 

323RUN datasette inspect {files} --inspect-file inspect-data.json 

324ENV PORT {port} 

325EXPOSE {port} 

326CMD {cmd}""".format( 

327 environment_variables="\n".join( 

328 [ 

329 "ENV {} '{}'".format(key, value) 

330 for key, value in environment_variables.items() 

331 ] 

332 ), 

333 files=" ".join(files), 

334 cmd=cmd, 

335 install_from=" ".join(install), 

336 spatialite_extras=SPATIALITE_DOCKERFILE_EXTRAS if spatialite else "", 

337 port=port, 

338 ).strip() 

339 

340 

341@contextmanager 

342def temporary_docker_directory( 

343 files, 

344 name, 

345 metadata, 

346 extra_options, 

347 branch, 

348 template_dir, 

349 plugins_dir, 

350 static, 

351 install, 

352 spatialite, 

353 version_note, 

354 secret, 

355 extra_metadata=None, 

356 environment_variables=None, 

357 port=8001, 

358): 

359 extra_metadata = extra_metadata or {} 

360 tmp = tempfile.TemporaryDirectory() 

361 # We create a datasette folder in there to get a nicer now deploy name 

362 datasette_dir = os.path.join(tmp.name, name) 

363 os.mkdir(datasette_dir) 

364 saved_cwd = os.getcwd() 

365 file_paths = [os.path.join(saved_cwd, file_path) for file_path in files] 

366 file_names = [os.path.split(f)[-1] for f in files] 

367 if metadata: 

368 metadata_content = parse_metadata(metadata.read()) 

369 else: 

370 metadata_content = {} 

371 # Merge in the non-null values in extra_metadata 

372 mergedeep.merge( 

373 metadata_content, 

374 {key: value for key, value in extra_metadata.items() if value is not None}, 

375 ) 

376 try: 

377 dockerfile = make_dockerfile( 

378 file_names, 

379 metadata_content and "metadata.json", 

380 extra_options, 

381 branch, 

382 template_dir, 

383 plugins_dir, 

384 static, 

385 install, 

386 spatialite, 

387 version_note, 

388 secret, 

389 environment_variables, 

390 port=port, 

391 ) 

392 os.chdir(datasette_dir) 

393 if metadata_content: 

394 open("metadata.json", "w").write(json.dumps(metadata_content, indent=2)) 

395 open("Dockerfile", "w").write(dockerfile) 

396 for path, filename in zip(file_paths, file_names): 

397 link_or_copy(path, os.path.join(datasette_dir, filename)) 

398 if template_dir: 

399 link_or_copy_directory( 

400 os.path.join(saved_cwd, template_dir), 

401 os.path.join(datasette_dir, "templates"), 

402 ) 

403 if plugins_dir: 

404 link_or_copy_directory( 

405 os.path.join(saved_cwd, plugins_dir), 

406 os.path.join(datasette_dir, "plugins"), 

407 ) 

408 for mount_point, path in static: 

409 link_or_copy_directory( 

410 os.path.join(saved_cwd, path), os.path.join(datasette_dir, mount_point) 

411 ) 

412 yield datasette_dir 

413 finally: 

414 tmp.cleanup() 

415 os.chdir(saved_cwd) 

416 

417 

418def detect_primary_keys(conn, table): 

419 " Figure out primary keys for a table. " 

420 table_info_rows = [ 

421 row 

422 for row in conn.execute('PRAGMA table_info("{}")'.format(table)).fetchall() 

423 if row[-1] 

424 ] 

425 table_info_rows.sort(key=lambda row: row[-1]) 

426 return [str(r[1]) for r in table_info_rows] 

427 

428 

429def get_outbound_foreign_keys(conn, table): 

430 infos = conn.execute("PRAGMA foreign_key_list([{}])".format(table)).fetchall() 

431 fks = [] 

432 for info in infos: 

433 if info is not None: 

434 id, seq, table_name, from_, to_, on_update, on_delete, match = info 

435 fks.append( 

436 {"column": from_, "other_table": table_name, "other_column": to_} 

437 ) 

438 return fks 

439 

440 

441def get_all_foreign_keys(conn): 

442 tables = [ 

443 r[0] for r in conn.execute('select name from sqlite_master where type="table"') 

444 ] 

445 table_to_foreign_keys = {} 

446 for table in tables: 

447 table_to_foreign_keys[table] = {"incoming": [], "outgoing": []} 

448 for table in tables: 

449 infos = conn.execute("PRAGMA foreign_key_list([{}])".format(table)).fetchall() 

450 for info in infos: 

451 if info is not None: 

452 id, seq, table_name, from_, to_, on_update, on_delete, match = info 

453 if table_name not in table_to_foreign_keys: 

454 # Weird edge case where something refers to a table that does 

455 # not actually exist 

456 continue 

457 table_to_foreign_keys[table_name]["incoming"].append( 

458 {"other_table": table, "column": to_, "other_column": from_} 

459 ) 

460 table_to_foreign_keys[table]["outgoing"].append( 

461 {"other_table": table_name, "column": from_, "other_column": to_} 

462 ) 

463 

464 return table_to_foreign_keys 

465 

466 

467def detect_spatialite(conn): 

468 rows = conn.execute( 

469 'select 1 from sqlite_master where tbl_name = "geometry_columns"' 

470 ).fetchall() 

471 return len(rows) > 0 

472 

473 

474def detect_fts(conn, table): 

475 "Detect if table has a corresponding FTS virtual table and return it" 

476 rows = conn.execute(detect_fts_sql(table)).fetchall() 

477 if len(rows) == 0: 

478 return None 

479 else: 

480 return rows[0][0] 

481 

482 

483def detect_fts_sql(table): 

484 return r""" 

485 select name from sqlite_master 

486 where rootpage = 0 

487 and ( 

488 sql like '%VIRTUAL TABLE%USING FTS%content="{table}"%' 

489 or sql like '%VIRTUAL TABLE%USING FTS%content=[{table}]%' 

490 or ( 

491 tbl_name = "{table}" 

492 and sql like '%VIRTUAL TABLE%USING FTS%' 

493 ) 

494 ) 

495 """.format( 

496 table=table 

497 ) 

498 

499 

500def detect_json1(conn=None): 

501 if conn is None: 

502 conn = sqlite3.connect(":memory:") 

503 try: 

504 conn.execute("SELECT json('{}')") 

505 return True 

506 except Exception: 

507 return False 

508 

509 

510def table_columns(conn, table): 

511 return [ 

512 r[1] 

513 for r in conn.execute( 

514 "PRAGMA table_info({});".format(escape_sqlite(table)) 

515 ).fetchall() 

516 ] 

517 

518 

519filter_column_re = re.compile(r"^_filter_column_\d+$") 

520 

521 

522def filters_should_redirect(special_args): 

523 redirect_params = [] 

524 # Handle _filter_column=foo&_filter_op=exact&_filter_value=... 

525 filter_column = special_args.get("_filter_column") 

526 filter_op = special_args.get("_filter_op") or "" 

527 filter_value = special_args.get("_filter_value") or "" 

528 if "__" in filter_op: 

529 filter_op, filter_value = filter_op.split("__", 1) 

530 if filter_column: 

531 redirect_params.append( 

532 ("{}__{}".format(filter_column, filter_op), filter_value) 

533 ) 

534 for key in ("_filter_column", "_filter_op", "_filter_value"): 

535 if key in special_args: 

536 redirect_params.append((key, None)) 

537 # Now handle _filter_column_1=name&_filter_op_1=contains&_filter_value_1=hello 

538 column_keys = [k for k in special_args if filter_column_re.match(k)] 

539 for column_key in column_keys: 

540 number = column_key.split("_")[-1] 

541 column = special_args[column_key] 

542 op = special_args.get("_filter_op_{}".format(number)) or "exact" 

543 value = special_args.get("_filter_value_{}".format(number)) or "" 

544 if "__" in op: 

545 op, value = op.split("__", 1) 

546 if column: 

547 redirect_params.append(("{}__{}".format(column, op), value)) 

548 redirect_params.extend( 

549 [ 

550 ("_filter_column_{}".format(number), None), 

551 ("_filter_op_{}".format(number), None), 

552 ("_filter_value_{}".format(number), None), 

553 ] 

554 ) 

555 return redirect_params 

556 

557 

558whitespace_re = re.compile(r"\s") 

559 

560 

561def is_url(value): 

562 "Must start with http:// or https:// and contain JUST a URL" 

563 if not isinstance(value, str): 

564 return False 

565 if not value.startswith("http://") and not value.startswith("https://"): 

566 return False 

567 # Any whitespace at all is invalid 

568 if whitespace_re.search(value): 

569 return False 

570 return True 

571 

572 

573css_class_re = re.compile(r"^[a-zA-Z]+[_a-zA-Z0-9-]*$") 

574css_invalid_chars_re = re.compile(r"[^a-zA-Z0-9_\-]") 

575 

576 

577def to_css_class(s): 

578 """ 

579 Given a string (e.g. a table name) returns a valid unique CSS class. 

580 For simple cases, just returns the string again. If the string is not a 

581 valid CSS class (we disallow - and _ prefixes even though they are valid 

582 as they may be confused with browser prefixes) we strip invalid characters 

583 and add a 6 char md5 sum suffix, to make sure two tables with identical 

584 names after stripping characters don't end up with the same CSS class. 

585 """ 

586 if css_class_re.match(s): 

587 return s 

588 md5_suffix = hashlib.md5(s.encode("utf8")).hexdigest()[:6] 

589 # Strip leading _, - 

590 s = s.lstrip("_").lstrip("-") 

591 # Replace any whitespace with hyphens 

592 s = "-".join(s.split()) 

593 # Remove any remaining invalid characters 

594 s = css_invalid_chars_re.sub("", s) 

595 # Attach the md5 suffix 

596 bits = [b for b in (s, md5_suffix) if b] 

597 return "-".join(bits) 

598 

599 

600def link_or_copy(src, dst): 

601 # Intended for use in populating a temp directory. We link if possible, 

602 # but fall back to copying if the temp directory is on a different device 

603 # https://github.com/simonw/datasette/issues/141 

604 try: 

605 os.link(src, dst) 

606 except OSError: 

607 shutil.copyfile(src, dst) 

608 

609 

610def link_or_copy_directory(src, dst): 

611 try: 

612 copytree(src, dst, copy_function=os.link, dirs_exist_ok=True) 

613 except OSError: 

614 copytree(src, dst, dirs_exist_ok=True) 

615 

616 

617def module_from_path(path, name): 

618 # Adapted from http://sayspy.blogspot.com/2011/07/how-to-import-module-from-just-file.html 

619 mod = types.ModuleType(name) 

620 mod.__file__ = path 

621 with open(path, "r") as file: 

622 code = compile(file.read(), path, "exec", dont_inherit=True) 

623 exec(code, mod.__dict__) 

624 return mod 

625 

626 

627async def resolve_table_and_format(table_and_format, table_exists, allowed_formats=[]): 

628 if "." in table_and_format: 

629 # Check if a table exists with this exact name 

630 it_exists = await table_exists(table_and_format) 

631 if it_exists: 

632 return table_and_format, None 

633 

634 # Check if table ends with a known format 

635 formats = list(allowed_formats) + ["csv", "jsono"] 

636 for _format in formats: 

637 if table_and_format.endswith(".{}".format(_format)): 

638 table = table_and_format[: -(len(_format) + 1)] 

639 return table, _format 

640 return table_and_format, None 

641 

642 

643def path_with_format(request, format, extra_qs=None): 

644 qs = extra_qs or {} 

645 path = request.path 

646 if "." in request.path: 

647 qs["_format"] = format 

648 else: 

649 path = "{}.{}".format(path, format) 

650 if qs: 

651 extra = urllib.parse.urlencode(sorted(qs.items())) 

652 if request.query_string: 

653 path = "{}?{}&{}".format(path, request.query_string, extra) 

654 else: 

655 path = "{}?{}".format(path, extra) 

656 elif request.query_string: 

657 path = "{}?{}".format(path, request.query_string) 

658 return path 

659 

660 

661class CustomRow(OrderedDict): 

662 # Loose imitation of sqlite3.Row which offers 

663 # both index-based AND key-based lookups 

664 def __init__(self, columns, values=None): 

665 self.columns = columns 

666 if values: 

667 self.update(values) 

668 

669 def __getitem__(self, key): 

670 if isinstance(key, int): 

671 return super().__getitem__(self.columns[key]) 

672 else: 

673 return super().__getitem__(key) 

674 

675 def __iter__(self): 

676 for column in self.columns: 

677 yield self[column] 

678 

679 

680def value_as_boolean(value): 

681 if value.lower() not in ("on", "off", "true", "false", "1", "0"): 

682 raise ValueAsBooleanError 

683 return value.lower() in ("on", "true", "1") 

684 

685 

686class ValueAsBooleanError(ValueError): 

687 pass 

688 

689 

690class WriteLimitExceeded(Exception): 

691 pass 

692 

693 

694class LimitedWriter: 

695 def __init__(self, writer, limit_mb): 

696 self.writer = writer 

697 self.limit_bytes = limit_mb * 1024 * 1024 

698 self.bytes_count = 0 

699 

700 async def write(self, bytes): 

701 self.bytes_count += len(bytes) 

702 if self.limit_bytes and (self.bytes_count > self.limit_bytes): 

703 raise WriteLimitExceeded( 

704 "CSV contains more than {} bytes".format(self.limit_bytes) 

705 ) 

706 await self.writer.write(bytes) 

707 

708 

709_infinities = {float("inf"), float("-inf")} 

710 

711 

712def remove_infinites(row): 

713 if any((c in _infinities) if isinstance(c, float) else 0 for c in row): 

714 return [None if (isinstance(c, float) and c in _infinities) else c for c in row] 

715 return row 

716 

717 

718class StaticMount(click.ParamType): 

719 name = "mount:directory" 

720 

721 def convert(self, value, param, ctx): 

722 if ":" not in value: 

723 self.fail( 

724 '"{}" should be of format mountpoint:directory'.format(value), 

725 param, 

726 ctx, 

727 ) 

728 path, dirpath = value.split(":", 1) 

729 dirpath = os.path.abspath(dirpath) 

730 if not os.path.exists(dirpath) or not os.path.isdir(dirpath): 

731 self.fail("%s is not a valid directory path" % value, param, ctx) 

732 return path, dirpath 

733 

734 

735def format_bytes(bytes): 

736 current = float(bytes) 

737 for unit in ("bytes", "KB", "MB", "GB", "TB"): 

738 if current < 1024: 

739 break 

740 current = current / 1024 

741 if unit == "bytes": 

742 return "{} {}".format(int(current), unit) 

743 else: 

744 return "{:.1f} {}".format(current, unit) 

745 

746 

747_escape_fts_re = re.compile(r'\s+|(".*?")') 

748 

749 

750def escape_fts(query): 

751 # If query has unbalanced ", add one at end 

752 if query.count('"') % 2: 

753 query += '"' 

754 bits = _escape_fts_re.split(query) 

755 bits = [b for b in bits if b and b != '""'] 

756 return " ".join( 

757 '"{}"'.format(bit) if not bit.startswith('"') else bit for bit in bits 

758 ) 

759 

760 

761class MultiParams: 

762 def __init__(self, data): 

763 # data is a dictionary of key => [list, of, values] or a list of [["key", "value"]] pairs 

764 if isinstance(data, dict): 

765 for key in data: 

766 assert isinstance( 

767 data[key], (list, tuple) 

768 ), "dictionary data should be a dictionary of key => [list]" 

769 self._data = data 

770 elif isinstance(data, list) or isinstance(data, tuple): 

771 new_data = {} 

772 for item in data: 

773 assert ( 

774 isinstance(item, (list, tuple)) and len(item) == 2 

775 ), "list data should be a list of [key, value] pairs" 

776 key, value = item 

777 new_data.setdefault(key, []).append(value) 

778 self._data = new_data 

779 

780 def __repr__(self): 

781 return "<MultiParams: {}>".format(self._data) 

782 

783 def __contains__(self, key): 

784 return key in self._data 

785 

786 def __getitem__(self, key): 

787 return self._data[key][0] 

788 

789 def keys(self): 

790 return self._data.keys() 

791 

792 def __iter__(self): 

793 yield from self._data.keys() 

794 

795 def __len__(self): 

796 return len(self._data) 

797 

798 def get(self, name, default=None): 

799 "Return first value in the list, if available" 

800 try: 

801 return self._data.get(name)[0] 

802 except (KeyError, TypeError): 

803 return default 

804 

805 def getlist(self, name): 

806 "Return full list" 

807 return self._data.get(name) or [] 

808 

809 

810class ConnectionProblem(Exception): 

811 pass 

812 

813 

814class SpatialiteConnectionProblem(ConnectionProblem): 

815 pass 

816 

817 

818def check_connection(conn): 

819 tables = [ 

820 r[0] 

821 for r in conn.execute( 

822 "select name from sqlite_master where type='table'" 

823 ).fetchall() 

824 ] 

825 for table in tables: 

826 try: 

827 conn.execute("PRAGMA table_info({});".format(escape_sqlite(table)),) 

828 except sqlite3.OperationalError as e: 

829 if e.args[0] == "no such module: VirtualSpatialIndex": 

830 raise SpatialiteConnectionProblem(e) 

831 else: 

832 raise ConnectionProblem(e) 

833 

834 

835class BadMetadataError(Exception): 

836 pass 

837 

838 

839def parse_metadata(content): 

840 # content can be JSON or YAML 

841 try: 

842 return json.loads(content) 

843 except json.JSONDecodeError: 

844 try: 

845 return yaml.safe_load(content) 

846 except yaml.YAMLError: 

847 raise BadMetadataError("Metadata is not valid JSON or YAML") 

848 

849 

850def _gather_arguments(fn, kwargs): 

851 parameters = inspect.signature(fn).parameters.keys() 

852 call_with = [] 

853 for parameter in parameters: 

854 if parameter not in kwargs: 

855 raise TypeError( 

856 "{} requires parameters {}, missing: {}".format( 

857 fn, tuple(parameters), set(parameters) - set(kwargs.keys()) 

858 ) 

859 ) 

860 call_with.append(kwargs[parameter]) 

861 return call_with 

862 

863 

864def call_with_supported_arguments(fn, **kwargs): 

865 call_with = _gather_arguments(fn, kwargs) 

866 return fn(*call_with) 

867 

868 

869async def async_call_with_supported_arguments(fn, **kwargs): 

870 call_with = _gather_arguments(fn, kwargs) 

871 return await fn(*call_with) 

872 

873 

874def actor_matches_allow(actor, allow): 

875 if actor is None and allow and allow.get("unauthenticated") is True: 

876 return True 

877 if allow is None: 

878 return True 

879 actor = actor or {} 

880 for key, values in allow.items(): 

881 if values == "*" and key in actor: 

882 return True 

883 if not isinstance(values, list): 

884 values = [values] 

885 actor_values = actor.get(key) 

886 if actor_values is None: 

887 continue 

888 if not isinstance(actor_values, list): 

889 actor_values = [actor_values] 

890 actor_values = set(actor_values) 

891 if actor_values.intersection(values): 

892 return True 

893 return False 

894 

895 

896async def check_visibility(datasette, actor, action, resource, default=True): 

897 "Returns (visible, private) - visible = can you see it, private = can others see it too" 

898 visible = await datasette.permission_allowed( 

899 actor, action, resource=resource, default=default, 

900 ) 

901 if not visible: 

902 return (False, False) 

903 private = not await datasette.permission_allowed( 

904 None, action, resource=resource, default=default, 

905 ) 

906 return visible, private 

907 

908 

909def resolve_env_secrets(config, environ): 

910 'Create copy that recursively replaces {"$env": "NAME"} with values from environ' 

911 if isinstance(config, dict): 

912 if list(config.keys()) == ["$env"]: 

913 return environ.get(list(config.values())[0]) 

914 elif list(config.keys()) == ["$file"]: 

915 return open(list(config.values())[0]).read() 

916 else: 

917 return { 

918 key: resolve_env_secrets(value, environ) 

919 for key, value in config.items() 

920 } 

921 elif isinstance(config, list): 

922 return [resolve_env_secrets(value, environ) for value in config] 

923 else: 

924 return config