1import asyncio 

2import uvicorn 

3import click 

4from click import formatting 

5from click_default_group import DefaultGroup 

6import json 

7import os 

8import pathlib 

9import shutil 

10from subprocess import call 

11import sys 

12from .app import Datasette, DEFAULT_CONFIG, CONFIG_OPTIONS, pm 

13from .utils import ( 

14 check_connection, 

15 parse_metadata, 

16 ConnectionProblem, 

17 SpatialiteConnectionProblem, 

18 temporary_docker_directory, 

19 value_as_boolean, 

20 StaticMount, 

21 ValueAsBooleanError, 

22) 

23 

24 

25class Config(click.ParamType): 

26 name = "config" 

27 

28 def convert(self, config, param, ctx): 

29 if ":" not in config: 

30 self.fail('"{}" should be name:value'.format(config), param, ctx) 

31 return 

32 name, value = config.split(":", 1) 

33 if name not in DEFAULT_CONFIG: 

34 self.fail( 

35 "{} is not a valid option (--help-config to see all)".format(name), 

36 param, 

37 ctx, 

38 ) 

39 return 

40 # Type checking 

41 default = DEFAULT_CONFIG[name] 

42 if isinstance(default, bool): 

43 try: 

44 return name, value_as_boolean(value) 

45 except ValueAsBooleanError: 

46 self.fail( 

47 '"{}" should be on/off/true/false/1/0'.format(name), param, ctx 

48 ) 

49 return 

50 elif isinstance(default, int): 

51 if not value.isdigit(): 

52 self.fail('"{}" should be an integer'.format(name), param, ctx) 

53 return 

54 return name, int(value) 

55 elif isinstance(default, str): 

56 return name, value 

57 else: 

58 # Should never happen: 

59 self.fail("Invalid option") 

60 

61 

62@click.group(cls=DefaultGroup, default="serve", default_if_no_args=True) 

63@click.version_option() 

64def cli(): 

65 """ 

66 Datasette! 

67 """ 

68 

69 

70@cli.command() 

71@click.argument("files", type=click.Path(exists=True), nargs=-1) 

72@click.option("--inspect-file", default="-") 

73@click.option( 

74 "sqlite_extensions", 

75 "--load-extension", 

76 envvar="SQLITE_EXTENSIONS", 

77 multiple=True, 

78 type=click.Path(exists=True, resolve_path=True), 

79 help="Path to a SQLite extension to load", 

80) 

81def inspect(files, inspect_file, sqlite_extensions): 

82 app = Datasette([], immutables=files, sqlite_extensions=sqlite_extensions) 

83 if inspect_file == "-": 

84 out = sys.stdout 

85 else: 

86 out = open(inspect_file, "w") 

87 loop = asyncio.get_event_loop() 

88 inspect_data = loop.run_until_complete(inspect_(files, sqlite_extensions)) 

89 out.write(json.dumps(inspect_data, indent=2)) 

90 

91 

92async def inspect_(files, sqlite_extensions): 

93 app = Datasette([], immutables=files, sqlite_extensions=sqlite_extensions) 

94 data = {} 

95 for name, database in app.databases.items(): 

96 counts = await database.table_counts(limit=3600 * 1000) 

97 data[name] = { 

98 "hash": database.hash, 

99 "size": database.size, 

100 "file": database.path, 

101 "tables": { 

102 table_name: {"count": table_count} 

103 for table_name, table_count in counts.items() 

104 }, 

105 } 

106 return data 

107 

108 

109@cli.group() 

110def publish(): 

111 "Publish specified SQLite database files to the internet along with a Datasette-powered interface and API" 

112 pass 

113 

114 

115# Register publish plugins 

116pm.hook.publish_subcommand(publish=publish) 

117 

118 

119@cli.command() 

120@click.option("--all", help="Include built-in default plugins", is_flag=True) 

121@click.option( 

122 "--plugins-dir", 

123 type=click.Path(exists=True, file_okay=False, dir_okay=True), 

124 help="Path to directory containing custom plugins", 

125) 

126def plugins(all, plugins_dir): 

127 "List currently available plugins" 

128 app = Datasette([], plugins_dir=plugins_dir) 

129 click.echo(json.dumps(app._plugins(all=all), indent=4)) 

130 

131 

132@cli.command() 

133@click.argument("files", type=click.Path(exists=True), nargs=-1, required=True) 

134@click.option( 

135 "-t", 

136 "--tag", 

137 help="Name for the resulting Docker container, can optionally use name:tag format", 

138) 

139@click.option( 

140 "-m", 

141 "--metadata", 

142 type=click.File(mode="r"), 

143 help="Path to JSON/YAML file containing metadata to publish", 

144) 

145@click.option("--extra-options", help="Extra options to pass to datasette serve") 

146@click.option("--branch", help="Install datasette from a GitHub branch e.g. master") 

147@click.option( 

148 "--template-dir", 

149 type=click.Path(exists=True, file_okay=False, dir_okay=True), 

150 help="Path to directory containing custom templates", 

151) 

152@click.option( 

153 "--plugins-dir", 

154 type=click.Path(exists=True, file_okay=False, dir_okay=True), 

155 help="Path to directory containing custom plugins", 

156) 

157@click.option( 

158 "--static", 

159 type=StaticMount(), 

160 help="Serve static files from this directory at /MOUNT/...", 

161 multiple=True, 

162) 

163@click.option( 

164 "--install", help="Additional packages (e.g. plugins) to install", multiple=True 

165) 

166@click.option("--spatialite", is_flag=True, help="Enable SpatialLite extension") 

167@click.option("--version-note", help="Additional note to show on /-/versions") 

168@click.option( 

169 "--secret", 

170 help="Secret used for signing secure values, such as signed cookies", 

171 envvar="DATASETTE_PUBLISH_SECRET", 

172 default=lambda: os.urandom(32).hex(), 

173) 

174@click.option( 

175 "-p", "--port", default=8001, help="Port to run the server on, defaults to 8001", 

176) 

177@click.option("--title", help="Title for metadata") 

178@click.option("--license", help="License label for metadata") 

179@click.option("--license_url", help="License URL for metadata") 

180@click.option("--source", help="Source label for metadata") 

181@click.option("--source_url", help="Source URL for metadata") 

182@click.option("--about", help="About label for metadata") 

183@click.option("--about_url", help="About URL for metadata") 

184def package( 

185 files, 

186 tag, 

187 metadata, 

188 extra_options, 

189 branch, 

190 template_dir, 

191 plugins_dir, 

192 static, 

193 install, 

194 spatialite, 

195 version_note, 

196 secret, 

197 port, 

198 **extra_metadata 

199): 

200 "Package specified SQLite files into a new datasette Docker container" 

201 if not shutil.which("docker"): 

202 click.secho( 

203 ' The package command requires "docker" to be installed and configured ', 

204 bg="red", 

205 fg="white", 

206 bold=True, 

207 err=True, 

208 ) 

209 sys.exit(1) 

210 with temporary_docker_directory( 

211 files, 

212 "datasette", 

213 metadata=metadata, 

214 extra_options=extra_options, 

215 branch=branch, 

216 template_dir=template_dir, 

217 plugins_dir=plugins_dir, 

218 static=static, 

219 install=install, 

220 spatialite=spatialite, 

221 version_note=version_note, 

222 secret=secret, 

223 extra_metadata=extra_metadata, 

224 port=port, 

225 ): 

226 args = ["docker", "build"] 

227 if tag: 

228 args.append("-t") 

229 args.append(tag) 

230 args.append(".") 

231 call(args) 

232 

233 

234@cli.command() 

235@click.argument("files", type=click.Path(exists=True), nargs=-1) 

236@click.option( 

237 "-i", 

238 "--immutable", 

239 type=click.Path(exists=True), 

240 help="Database files to open in immutable mode", 

241 multiple=True, 

242) 

243@click.option( 

244 "-h", 

245 "--host", 

246 default="127.0.0.1", 

247 help=( 

248 "Host for server. Defaults to 127.0.0.1 which means only connections " 

249 "from the local machine will be allowed. Use 0.0.0.0 to listen to " 

250 "all IPs and allow access from other machines." 

251 ), 

252) 

253@click.option( 

254 "-p", 

255 "--port", 

256 default=8001, 

257 help="Port for server, defaults to 8001. Use -p 0 to automatically assign an available port.", 

258) 

259@click.option( 

260 "--debug", is_flag=True, help="Enable debug mode - useful for development" 

261) 

262@click.option( 

263 "--reload", 

264 is_flag=True, 

265 help="Automatically reload if database or code change detected - useful for development", 

266) 

267@click.option( 

268 "--cors", is_flag=True, help="Enable CORS by serving Access-Control-Allow-Origin: *" 

269) 

270@click.option( 

271 "sqlite_extensions", 

272 "--load-extension", 

273 envvar="SQLITE_EXTENSIONS", 

274 multiple=True, 

275 type=click.Path(exists=True, resolve_path=True), 

276 help="Path to a SQLite extension to load", 

277) 

278@click.option( 

279 "--inspect-file", help='Path to JSON file created using "datasette inspect"' 

280) 

281@click.option( 

282 "-m", 

283 "--metadata", 

284 type=click.File(mode="r"), 

285 help="Path to JSON/YAML file containing license/source metadata", 

286) 

287@click.option( 

288 "--template-dir", 

289 type=click.Path(exists=True, file_okay=False, dir_okay=True), 

290 help="Path to directory containing custom templates", 

291) 

292@click.option( 

293 "--plugins-dir", 

294 type=click.Path(exists=True, file_okay=False, dir_okay=True), 

295 help="Path to directory containing custom plugins", 

296) 

297@click.option( 

298 "--static", 

299 type=StaticMount(), 

300 help="Serve static files from this directory at /MOUNT/...", 

301 multiple=True, 

302) 

303@click.option("--memory", is_flag=True, help="Make :memory: database available") 

304@click.option( 

305 "--config", 

306 type=Config(), 

307 help="Set config option using configname:value datasette.readthedocs.io/en/latest/config.html", 

308 multiple=True, 

309) 

310@click.option( 

311 "--secret", 

312 help="Secret used for signing secure values, such as signed cookies", 

313 envvar="DATASETTE_SECRET", 

314) 

315@click.option( 

316 "--root", 

317 help="Output URL that sets a cookie authenticating the root user", 

318 is_flag=True, 

319) 

320@click.option("--version-note", help="Additional note to show on /-/versions") 

321@click.option("--help-config", is_flag=True, help="Show available config options") 

322def serve( 

323 files, 

324 immutable, 

325 host, 

326 port, 

327 debug, 

328 reload, 

329 cors, 

330 sqlite_extensions, 

331 inspect_file, 

332 metadata, 

333 template_dir, 

334 plugins_dir, 

335 static, 

336 memory, 

337 config, 

338 secret, 

339 root, 

340 version_note, 

341 help_config, 

342 return_instance=False, 

343): 

344 """Serve up specified SQLite database files with a web UI""" 

345 if help_config: 

346 formatter = formatting.HelpFormatter() 

347 with formatter.section("Config options"): 

348 formatter.write_dl( 

349 [ 

350 (option.name, "{} (default={})".format(option.help, option.default)) 

351 for option in CONFIG_OPTIONS 

352 ] 

353 ) 

354 click.echo(formatter.getvalue()) 

355 sys.exit(0) 

356 if reload: 

357 import hupper 

358 

359 reloader = hupper.start_reloader("datasette.cli.serve") 

360 if immutable: 

361 reloader.watch_files(immutable) 

362 if metadata: 

363 reloader.watch_files([metadata.name]) 

364 

365 inspect_data = None 

366 if inspect_file: 

367 inspect_data = json.load(open(inspect_file)) 

368 

369 metadata_data = None 

370 if metadata: 

371 metadata_data = parse_metadata(metadata.read()) 

372 

373 kwargs = dict( 

374 immutables=immutable, 

375 cache_headers=not debug and not reload, 

376 cors=cors, 

377 inspect_data=inspect_data, 

378 metadata=metadata_data, 

379 sqlite_extensions=sqlite_extensions, 

380 template_dir=template_dir, 

381 plugins_dir=plugins_dir, 

382 static_mounts=static, 

383 config=dict(config), 

384 memory=memory, 

385 secret=secret, 

386 version_note=version_note, 

387 ) 

388 

389 # if files is a single directory, use that as config_dir= 

390 if 1 == len(files) and os.path.isdir(files[0]): 

391 kwargs["config_dir"] = pathlib.Path(files[0]) 

392 files = [] 

393 

394 ds = Datasette(files, **kwargs) 

395 

396 if return_instance: 

397 # Private utility mechanism for writing unit tests 

398 return ds 

399 

400 # Run the "startup" plugin hooks 

401 asyncio.get_event_loop().run_until_complete(ds.invoke_startup()) 

402 

403 # Run async sanity checks - but only if we're not under pytest 

404 asyncio.get_event_loop().run_until_complete(check_databases(ds)) 

405 

406 # Start the server 

407 if root: 

408 print("http://{}:{}/-/auth-token?token={}".format(host, port, ds._root_token)) 

409 uvicorn.run(ds.app(), host=host, port=port, log_level="info") 

410 

411 

412async def check_databases(ds): 

413 # Run check_connection against every connected database 

414 # to confirm they are all usable 

415 for database in list(ds.databases.values()): 

416 try: 

417 await database.execute_fn(check_connection) 

418 except SpatialiteConnectionProblem: 

419 raise click.UsageError( 

420 "It looks like you're trying to load a SpatiaLite" 

421 " database without first loading the SpatiaLite module." 

422 "\n\nRead more: https://datasette.readthedocs.io/en/latest/spatialite.html" 

423 ) 

424 except ConnectionProblem as e: 

425 raise click.UsageError( 

426 "Connection to {} failed check: {}".format( 

427 database.path, str(e.args[0]) 

428 ) 

429 )