1. #!/usr/bin/env python
  2. # encoding: utf-8
  3.  
  4. """static
  5.  
  6. Create and/or upload a static copy of the repository.
  7.  
  8. The main goal is sharing Mercurial on servers with only FTP access and
  9. statically served files, while providing the same information as hg
  10. serve and full solutions like bitbucket and gitorious (naturally
  11. without the interactivity).
  12. """
  13.  
  14. __plan__ = """
  15.  
  16. * Create the static-dir in the repo:
  17.   - Overview: Readme + commits + template ✔
  18.   - Changes: Commit-Log + each commit as commit/<hex> ✔
  19.   - source: a filetree, shown as sourcecode: src/<hex>/<path> and raw/<hex>/<path>
  20.     as HTML (src) and as plain files (raw)
  21.     file-links in revision infos: link to the latest change in the file.
  22.     → manifest ⇒ filelog ⇒ revision before the current.
  23.     src/<hex>/index.html: manifest with file-links. ✔
  24.   - Upload it to ftp ✔
  25.  
  26.   - Add a list of branches, heads and tags to the summary page.
  27.   - if b is used: a bugtracker: issue/<id>/<name>
  28.   - fork-/clone-info for each entry in [paths] with its incoming data (if it has some):
  29.     clone/<pathname>/ → incoming log (commits) + possibly an associated issue in b.
  30.   - More complex Readme parsing.
  31.   - add linenumbers to the src files.
  32.   - add sourcecode coloring to the src files.
  33.   - Treat branch heads specially: link on the main page.
  34.  
  35. * Usage:
  36.   - hg static [--name] [-r] [folder] → parse the static folder for the current revision.
  37.     Mimic pull and clone wherever possible: This is a clone to <repo>/static
  38.   - hg static --upload <FTP-path> [folder] → update and upload the folder == clone/push
  39.  
  40. * Idea: hg clone/push ftp://host.tld/path/to/repo → hg static --upload
  41.  
  42. * Setup a new static repo or update an existing one: hg static --upload ftp://host.tld/path/to/repo
  43.  
  44. """
  45.  
  46. import os
  47. from os.path import join, isdir, isfile, basename, dirname
  48. import shutil
  49. import re
  50. import mercurial
  51. import ftplib
  52. import socket
  53. import datetime
  54. from mercurial import cmdutil
  55. from mercurial import commands
  56.  
  57. _staticidentifier = ".statichgrepo"
  58.  
  59. templates = {
  60.     "head": """<!DOCTYPE html>
  61. <html><head>
  62.     <meta charset="utf-8" />
  63.     <meta http-equiv="Content-Type" content="text/html; charset=utf-8" /> <!--duplicate for older browsers-->
  64.     <link rel="stylesheet" href="style.css" type="text/css" media="screen" />
  65.     <link rel="stylesheet" href="print.css" type="text/css" media="print" />
  66.     <title>{reponame}</title>
  67. </head>
  68. <body>
  69. <h1>{reponame}</h1>
  70. """,
  71.     "srchead": """<!DOCTYPE html>
  72. <html><head>
  73.     <meta charset="utf-8" />
  74.     <meta http-equiv="Content-Type" content="text/html; charset=utf-8" /> <!--duplicate for older browsers-->
  75.     <link rel="stylesheet" href="style.css" type="text/css" media="screen" />
  76.     <link rel="stylesheet" href="print.css" type="text/css" media="print" />
  77.     <title>{filetitle}</title>
  78. </head>
  79. <body>
  80. """,
  81.     "foot": "</body></html>",
  82.     "screenstyle": """ """,
  83.     "printstyle": """ """,
  84.     "manifesthead": """<h2>Commit: <a href='../../commit/{hex}.html'>{hex}</a></h2>
  85. <p>{desc}</p><p>{user}</p>
  86.     <h2>Files in this revision</h2>"""
  87.  
  88. }
  89.  
  90. _indexregexp = re.compile("^\\.*index.html$")
  91.  
  92.  
  93. def parsereadme(filepath):
  94.     """Parse the readme file"""
  95.     with open(filepath) as r:
  96.         return "<pre>" + r.read() + "</pre>"
  97.  
  98.  
  99. def writeoverview(ui, repo, target, name):
  100.     """Create the overview page"""
  101.     overview = open(join(target, "index.html"), "w")
  102.     overview.write(templates["head"].replace("{reponame}", name))
  103.     # add a short identifier from the first line of the readme, if it
  104.     # exists # TODO: Parse different types of readme files
  105.     readme = name
  106.     for f in os.listdir(repo.root):
  107.         if f.lower().startswith("readme"):
  108.             readme = parsereadme(f)
  109.             overview.write( "\n".join(readme.splitlines()[:3]))
  110.             break
  111.     # now the links to the log and the files.
  112.     overview.write("<p><a href='commits'>changelog</a> | <a href='src/" + repo["tip"].hex() + "/'>files</a></p>")
  113.     # now add the 5 most recent log entries
  114.     # divert all following ui output to a string, so we can just use standard functions
  115.     overview.write("<h2>Changes (<a href='commits'>full changelog</a>)</h2>")
  116.     ui.pushbuffer()
  117.     t = cmdutil.changeset_templater(ui, repo, patch=False, diffopts=None, mapfile=None, buffered=False)
  118.     t.use_template("""<div style='float: right; padding-left: 0.5em'><em>({author|person})</em></div><strong> {date|shortdate}: <a href='commit/{node}.html'>{desc|strip|fill68|firstline}</a> <span style='font-size: xx-small'>{branches} {tags}</span><p>{desc|escape}</p>""")
  119.     for c in range(1, min(len(repo.changelog), 5)):
  120.         ctx = repo.changectx(str(-c))
  121.         t.show(ctx)
  122.     overview.write(ui.popbuffer())
  123.  
  124.     # add the full readme
  125.     overview.write("<h2>Readme</h2>")
  126.     overview.write(readme)
  127.  
  128.     # finish the overview
  129.     overview.write(templates["foot"])
  130.  
  131. def writelog(ui, repo, target, name):
  132.     """Write the full changelog, in steps of 100."""
  133.     commits = join(target, "commits")
  134.  
  135.     # create the folders
  136.     if not isdir(commits):
  137.         os.makedirs(commits)
  138.     for i in range(len(repo.changelog)/100):
  139.         d = commits+"-"+str(i+1)+"00"
  140.         if not isdir(d):
  141.             os.makedirs(d)
  142.  
  143.     # create the log files
  144.     t = cmdutil.changeset_templater(ui, repo, patch=False, diffopts=None, mapfile=None, buffered=False)
  145.     t.use_template("""<div style='float: right; padding-left: 0.5em'><em>({author|person})</em></div><strong> {date|shortdate}: <a href='../commit/{node}.html'>{desc|strip|fill68|firstline}</a> <span style='font-size: xx-small'>{branches} {tags}</span><p>{desc|escape}</p>""")
  146.     logs = []
  147.     for ck in range(0, len(repo.changelog)/100+1):
  148.         ui.pushbuffer()
  149.         if ck:
  150.             dd = d
  151.             di = str(ck)+"00"
  152.             d = commits+"-"+di
  153.             logs[-1].write("<p><a href=\"../commits-"+di+"\">earlier</a></p>")
  154.             if ck>2:
  155.                 # the older log gets a reference to the newer one
  156.                 logs[-1].write("<p><a href=\"../commits-"+str(ck-2)+"00"+"\">later</a></p>")
  157.             elif ck>1:
  158.                 logs[-1].write("<p><a href=\"../commits\">later</a></p>")
  159.             logs.append(open(join(d, "index.html"), "w"))
  160.         else:
  161.             d = commits
  162.             logs.append(open(join(d, "index.html"), "w"))
  163.  
  164.         logs[-1].write(templates["head"].replace("{reponame}", "<a href='../'>"+name+"</a>"))
  165.         for c in range(ck*100+1, min(len(repo.changelog), (ck+1)*100)):
  166.             ctx = repo.changectx(str(-c))
  167.             t.show(ctx)
  168.         logs[-1].write(ui.popbuffer())
  169.  
  170.     for l in logs:
  171.         l.write(templates["foot"].replace("{reponame}", "<a href='../'>"+name+"</a>"))
  172.         l.close()
  173.  
  174.  
  175. def writecommits(ui, repo, target, name, force=False):
  176.     """Write all not yet existing commit files."""
  177.     commit = join(target, "commit")
  178.  
  179.     # create the folders
  180.     if not isdir(commit):
  181.         os.makedirs(commit)
  182.  
  183.     t = cmdutil.changeset_templater(ui, repo, patch=False, diffopts=None, mapfile=None, buffered=False)
  184.     t.use_template("""<div style='float: right; padding-left: 0.5em'><em>({author|person})</em></div><strong> {date|shortdate}: <a href='../src/{node}/'>{desc|strip|fill68|firstline}</a> <span style='font-size: xx-small'>{branches} {tags}</span><p>{desc|escape}</p>""")
  185.     for c in range(len(repo.changelog)):
  186.         ctx = repo.changectx(str(c))
  187.         cpath = join(commit, ctx.hex() + ".html")
  188.         if not force and isfile(cpath):
  189.             continue
  190.         with open(cpath, "w") as cf:
  191.             cf.write(templates["head"].replace("{reponame}", "<a href='../'>"+name+"</a>"))
  192.             ui.pushbuffer()
  193.             t.show(ctx)
  194.             cf.write(ui.popbuffer())
  195.             ui.pushbuffer()
  196.             commands.diff(ui, repo, change=str(c), git=True)
  197.             cf.write("<pre>"+ui.popbuffer().replace("<", "<")+"</pre>")
  198.             cf.write(templates["foot"].replace("{reponame}", "<a href='../'>"+name+"</a>"))
  199.  
  200.  
  201. def escapename(filename):
  202.     """escape index.html as .index.html and .ind… as ..ind… and so fort."""
  203.     if _indexregexp.match(filename) is not None:
  204.         return "." + filename
  205.     else: return filename
  206.  
  207.  
  208. def parsesrcdata(data):
  209.     """Parse a src file into a html file."""
  210.     return "<pre>"+data.replace("<", "<")+"</pre>"
  211.  
  212. def srcpath(target, ctx, filename):
  213.     """Get the relative path to the static sourcefile for an already escaped filename."""
  214.     return join(target,"src",ctx.hex(),filename+".html")
  215.  
  216. def rawpath(target, ctx, filename):
  217.     """Get the relative path to the static sourcefile for an already escaped filename."""
  218.     return join(target,"raw",ctx.hex(),filename)
  219.  
  220. def createindex(target, ctx):
  221.     """Create an index page for the changecontext: the commit message + the user + all files in the changecontext."""
  222.     # first the head
  223.     index = templates["manifesthead"].replace(
  224.         "{hex}", ctx.hex()).replace(
  225.         "{desc}", ctx.description()).replace(
  226.             "{user}", ctx.user())
  227.     # then the files
  228.     index += "<ul>"
  229.     for filename in ctx:
  230.         filectx = ctx[filename]
  231.         lasteditctx = filectx.filectx(filectx.filerev())
  232.         index += "<li><a href='../../"+ join("src",lasteditctx.hex(), escapename(filename)+".html") + "'>" + filename + "</a>"# (<a href='../../" + join("raw",lasteditctx.hex(), filename) + "'>raw</a>)</li>"
  233.     index += "</ul>"
  234.     return index
  235.  
  236. def writesourcetree(ui, repo, target, name, force):
  237.     """Write manifests for all commits and websites for all files.
  238.  
  239.     * For each file, write sites for all revisions where the file was changed: under src/<hex>/path as html site (with linenumbers and maybe colored source), under raw/<hex>/<path> as plain files. If there is an index.html file, write it as .index.html. If there also is .index.html, turn it to ..index.html, …
  240.     * For each commit write an index with links to the included files at their latest revisions before/at the commit.
  241.     """
  242.     # first write all files in all commits.
  243.     for c in range(len(repo.changelog)):
  244.         ctx = repo.changectx(str(c))
  245.         for filename in ctx.files():
  246.             filectx = ctx[filename]
  247.             # first write the raw data
  248.             filepath = rawpath(target,ctx,filectx.path())
  249.             # skip already existing files
  250.             if not force and isfile(filepath):
  251.                 continue
  252.             try:
  253.                 os.makedirs(dirname(filepath))
  254.             except OSError: pass # exists
  255.             with open(filepath, "w") as f:
  256.                 f.write(filectx.data())
  257.             # then write it as html
  258.             _filenameescaped = escapename(filectx.path())
  259.             filepath = srcpath(target,ctx,_filenameescaped)
  260.             try:
  261.                 os.makedirs(dirname(filepath))
  262.             except OSError: pass # exists
  263.             with open(filepath, "w") as f:
  264.                 f.write(templates["srchead"].replace("{filetitle}", name+": " + filename))
  265.                 f.write(parsesrcdata(filectx.data()))
  266.                 f.write(templates["foot"].replace("{reponame}", name))
  267.     # then write manifests for all commits
  268.     for c in range(len(repo.changelog)):
  269.         ctx = repo.changectx(str(c))
  270.         filepath = join(target,"src",ctx.hex(),"index.html")
  271.         # skip already existing files
  272.         if not force and isfile(filepath):
  273.             continue
  274.         try:
  275.             os.makedirs(dirname(filepath))
  276.         except OSError: pass # exists
  277.         with open(filepath, "w") as f:
  278.             f.write(templates["head"].replace("{reponame}", "<a href='../../'>"+name+"</a>"))
  279.             f.write(createindex(target, ctx))
  280.             f.write(templates["foot"].replace("{reponame}", "<a href='../../'>"+name+"</a>"))
  281.  
  282.  
  283. def parsesite(ui, repo, target, **opts):
  284.     """Create the static folder."""
  285.     idfile = join(target, _staticidentifier)
  286.     if not isdir(target):
  287.         # make sure the target exists
  288.         os.makedirs(target)
  289.     else: # make sure it is a staticrepo
  290.         if not isfile(idfile):
  291.             if not ui.prompt("The target folder exists is no static repo. Really use it?", default="n").lower() in ["y", "yes"]:
  292.                 return
  293.     with open(idfile, "w") as i:
  294.         i.write("")
  295.  
  296.     if opts["name"]:
  297.         name = opts["name"]
  298.     elif target != "static": name = target
  299.     else: name = basename(repo.root)
  300.  
  301.     # first the stylesheets
  302.     screenstyle = opts["screenstyle"]
  303.     if screenstyle:
  304.         shutil.copyfile(screenstyle, join(target, "style.css"))
  305.     else:
  306.         with open(join(target, "style.css"), "w") as f:
  307.             f.write(templates["screenstyle"])
  308.     printstyle = opts["printstyle"]
  309.     if printstyle:
  310.         shutil.copyfile(printstyle, join(target, "print.css"))
  311.     else:
  312.         with open(join(target, "print.css"), "w") as f:
  313.             f.write(templates["printstyle"])
  314.  
  315.     # then the overview
  316.     writeoverview(ui, repo, target, name)
  317.  
  318.     # and the log
  319.     writelog(ui, repo, target, name)
  320.  
  321.     # and all commit files
  322.     writecommits(ui, repo, target, name, force=opts["force"])
  323.  
  324.     # and all file data
  325.     writesourcetree(ui, repo, target, name, force=opts["force"])
  326.  
  327.  
  328. def addrepo(ui, repo, target):
  329.     """Add the repo to the target and make sure it is up to date."""
  330.     try:
  331.         commands.init(ui, dest=target)
  332.     except mercurial.error.RepoError, e:
  333.         # already exists
  334.         pass
  335.     ui.pushbuffer()
  336.     commands.push(ui, repo, dest=target)
  337.     ui.popbuffer()
  338.  
  339.  
  340. def upload(ui, repo, target, ftpstring, force):
  341.     """upload the repo to the FTP server identified by the ftp string."""
  342.     user, password = ftpstring.split("@")[0].split(":")
  343.     serverandpath = "@".join(ftpstring.split("@")[1:])
  344.     server = serverandpath.split("/")[0]
  345.     ftppath = "/".join(serverandpath.split("/")[1:])
  346.     timeout = 10
  347.     try:
  348.         ftp = ftplib.FTP(server, user, password, "", timeout)
  349.     except socket.timeout:
  350.         ui.warn("connection to ", server, " timed out after ", timeout, " seconds.\n")
  351.         return
  352.  
  353.     ui.status(ftp.getwelcome(), "\n")
  354.  
  355.     # create the target dir.
  356.     serverdir = dirname(ftppath)
  357.     serverdirparts = ftppath.split("/")
  358.     sd = serverdirparts[0]
  359.     if not sd in ftp.nlst():
  360.         ftp.mkd(sd)
  361.     for sdp in serverdirparts[1:]:
  362.         sdo = sd
  363.         sd = os.path.join(sd, sdp)
  364.         if not sd in ftp.nlst(sdo):
  365.             ftp.mkd(sd)
  366.  
  367.  
  368.     ftp.cwd(ftppath)
  369.     if not ftp.pwd() == "/" + ftppath:
  370.         ui.warn("not in the correct ftp directory. Cowardly bailing out.\n")
  371.         return
  372.  
  373.     #ftp.dir()
  374.     #return
  375.     ftpfeatures = ftp.sendcmd("FEAT")
  376.  
  377.     _ftpdircache = set()
  378.  
  379.     for d, dirnames, filenames in os.walk(target):
  380.         for filename in filenames:
  381.             localfile = join(d, filename)
  382.             serverfile = localfile[len(target)+1:]
  383.             serverdir = dirname(serverfile)
  384.             serverdirparts = serverdir.split("/")
  385. #            print serverdirparts, serverfile
  386.             with open(localfile, "rb") as f:
  387.                 sd = serverdirparts[0]
  388.                 if sd and not sd in _ftpdircache and not sd in ftp.nlst():
  389.                     try:
  390.                         ui.status("creating directory ", sd, "\n")
  391.                         ftp.mkd(sd)
  392.                         _ftpdircache.add(sd)
  393.                     except ftplib.error_perm, resp:
  394.                         ui.warn("could not create directory ", sd, ": " , resp, "\n")
  395.                     else: _ftpdircache.add(sd)
  396.  
  397.                 for sdp in serverdirparts[1:]:
  398.                     sdold = sd
  399.                     sd = join(sd, sdp)
  400.                     #print sd, sdp
  401.                     #print ftp.nlst(sdold)
  402.                     if sd and not sd in _ftpdircache and not sd in ftp.nlst(sdold):
  403.                         try:
  404.                             ui.status("creating directory ", sd, "\n")
  405.                             ftp.mkd(sd)
  406.                             _ftpdircache.add(sd)
  407.                         except ftplib.error_perm, resp:
  408.                             ui.warn("could not create directory ", sd, ": " , resp, "\n")
  409.                     else: _ftpdircache.add(sd)
  410.  
  411.  
  412.                 if not serverfile in ftp.nlst(serverdir) or force:
  413.                     if force:
  414.                         ui.status("uploading ", serverfile, " because I am forced to.\n")
  415.                     else:
  416.                         ui.status("uploading ", serverfile, " because it is not yet online.\n")
  417.  
  418.                     ftp.storbinary("STOR "+ serverfile, f)
  419.                 else:
  420.                     # reupload the file if the file on the server is older than the local file.
  421.                     if " MDTM" in ftpfeatures.splitlines():
  422.                         ftpmtime = ftp.sendcmd("MDTM " + serverfile).split()[1]
  423.                         localmtime = os.stat(localfile).st_mtime
  424.                         localmtimestr = datetime.datetime.utcfromtimestamp(localmtime).strftime("%Y%m%d%H%M%S")
  425.                         newer = int(localmtimestr) > int(ftpmtime)
  426.                         if newer:
  427.                             ui.status("uploading ", serverfile, " because it is newer than the file on the FTP server.\n")
  428.                             ftp.storbinary("STOR "+ serverfile, f)
  429.  
  430.  
  431.  
  432. def static(ui, repo, target=None, **opts):
  433.     """Create a static copy of the repository and/or upload it to an FTP server."""
  434.     if repo.root == target:
  435.         ui.warn("static target repo can’t be the current repo")
  436.         return
  437.     # first: just create the site.
  438.     if not target: target = "static"
  439.     parsesite(ui, repo, target, **opts)
  440.     # add the hg repo to the static site
  441.     addrepo(ui, repo, target)
  442.     if opts["upload"]:
  443.         # upload the repo
  444.         upload(ui, repo, target, opts["upload"], opts["force"])
  445.  
  446.  
  447.  
  448. cmdtable = {
  449.     # "command-name": (function-call, options-list, help-string)
  450.     "static": (static,
  451.                      [
  452.                       #('r', 'rev', None, 'parse the given revision'),
  453.                       #('a', 'all', None, 'parse all revisions (requires much space)'),
  454.                       ('n', 'name', "", 'the repo name. Default: folder or last segment of the repo-path.'),
  455.                       ('u', 'upload', "", 'upload the repo to the given ftp host. Format: user:password@host/path/to/dir'),
  456.                       ('f', 'force', False, 'force recreating all commit files. Slow.'),
  457.                       ('s', 'screenstyle', "", 'use a custom stylesheet for display on screen'),
  458.                       ('p', 'printstyle', "", 'use a custom stylesheet for printing')],
  459.                      "[options] [folder]")
  460. }