1. #!/usr/bin/env python
  2. # encoding: utf-8
  3.  
  4. """static
  5.  
  6. Create and/or upload a static copy of the repository.
  7.  
  8. The main goal is sharing Mercurial on servers with only FTP access and
  9. statically served files, while providing the same information as hg
  10. serve and full solutions like bitbucket and gitorious (naturally
  11. without the interactivity).
  12. """
  13.  
  14. __plan__ = """
  15.  
  16. * Create the static-dir in the repo:
  17.   - Overview: Readme + commits + template ✔
  18.   - Changes: Commit-Log + each commit as commit/<hex> ✔
  19.   - source: a filetree, shown as sourcecode: src/<hex>/<path> and raw/<hex>/<path>
  20.     as HTML (src) and as plain files (raw)
  21.     file-links in revision infos: link to the latest change in the file.
  22.     → manifest ⇒ filelog ⇒ revision before the current.
  23.     src/<hex>/index.html: manifest with file-links. ✔
  24.   - Upload it to ftp ✔
  25.  
  26.   - Add a list of branches, heads and tags to the summary page.
  27.   - if b is used: a bugtracker: issue/<id>/<name>
  28.   - fork-/clone-info for each entry in [paths] with its incoming data (if it has some):
  29.     clone/<pathname>/ → incoming log (commits) + possibly an associated issue in b.
  30.   - More complex Readme parsing.
  31.   - add linenumbers to the src files.
  32.   - add sourcecode coloring to the src files.
  33.   - Treat branch heads specially: link on the main page.
  34.  
  35. * Usage:
  36.   - hg static [--name] [-r] [folder] → parse the static folder for the current revision.
  37.     Mimic pull and clone wherever possible: This is a clone to <repo>/static
  38.   - hg static --upload <FTP-path> [folder] → update and upload the folder == clone/push
  39.  
  40. * Idea: hg clone/push ftp://host.tld/path/to/repo → hg static --upload
  41.  
  42. * Setup a new static repo or update an existing one: hg static --upload ftp://host.tld/path/to/repo
  43.  
  44. """
  45.  
  46. import os
  47. from os.path import join, isdir, isfile, basename, dirname
  48. import shutil
  49. import re
  50. import mercurial
  51. import ftplib
  52. import socket
  53. import datetime
  54. from mercurial import cmdutil
  55. from mercurial import commands
  56.  
  57. _staticidentifier = ".statichgrepo"
  58.  
  59. templates = {
  60.     "head": """<!DOCTYPE html>
  61. <html><head>
  62.     <meta charset="utf-8" />
  63.     <meta http-equiv="Content-Type" content="text/html; charset=utf-8" /> <!--duplicate for older browsers-->
  64.     <link rel="stylesheet" href="style.css" type="text/css" media="screen" />
  65.     <link rel="stylesheet" href="print.css" type="text/css" media="print" />
  66.     <title>{reponame}</title>
  67. </head>
  68. <body>
  69. <h1>{reponame}</h1>
  70. """,
  71.     "foot": "</body></html>",
  72.     "screenstyle": """ """,
  73.     "printstyle": """ """,
  74.     "manifesthead": """<h2>Commit: {hex} </h2>
  75. <p>{desc}</p><p>{user}</p>
  76.     <h2>Files changed</h2>"""
  77.  
  78. }
  79.  
  80. _indexregexp = re.compile("^\\.*index.html$")
  81.  
  82.  
  83. def parsereadme(filepath):
  84.     """Parse the readme file"""
  85.     with open(filepath) as r:
  86.         return "<pre>" + r.read() + "</pre>"
  87.  
  88.  
  89. def writeoverview(ui, repo, target, name):
  90.     """Create the overview page"""
  91.     overview = open(join(target, "index.html"), "w")
  92.     overview.write(templates["head"].replace("{reponame}", name))
  93.     # add a short identifier from the first line of the readme, if it
  94.     # exists # TODO: Parse different types of readme files
  95.     readme = name
  96.     for f in os.listdir(repo.root):
  97.         if f.lower().startswith("readme"):
  98.             readme = parsereadme(f)
  99.             overview.write( "\n".join(readme.splitlines()[:3]))
  100.             break
  101.     # now the links to the log and the files.
  102.     overview.write("<p><a href='commits'>changelog</a> | <a href='src'>files</a></p>")
  103.     # now add the 5 most recent log entries
  104.     # divert all following ui output to a string, so we can just use standard functions
  105.     overview.write("<h2>Changes (<a href='commits'>full changelog</a>)</h2>")
  106.     ui.pushbuffer()
  107.     t = cmdutil.changeset_templater(ui, repo, patch=False, diffopts=None, mapfile=None, buffered=False)
  108.     t.use_template("""<div style='float: right; padding-left: 0.5em'><em>({author|person})</em></div><strong> {date|shortdate}: <a href='commit/{node}.html'>{desc|strip|fill68|firstline}</a> <span style='font-size: xx-small'>{branches} {tags}</span><p>{desc|escape}</p>""")
  109.     for c in range(1, min(len(repo.changelog), 5)):
  110.         ctx = repo.changectx(str(-c))
  111.         t.show(ctx)
  112.     overview.write(ui.popbuffer())
  113.  
  114.     # add the full readme
  115.     overview.write("<h2>Readme</h2>")
  116.     overview.write(readme)
  117.  
  118.     # finish the overview
  119.     overview.write(templates["foot"])
  120.  
  121. def writelog(ui, repo, target, name):
  122.     """Write the full changelog, in steps of 100."""
  123.     commits = join(target, "commits")
  124.  
  125.     # create the folders
  126.     if not isdir(commits):
  127.         os.makedirs(commits)
  128.     for i in range(len(repo.changelog)/100):
  129.         d = commits+"-"+str(i+1)+"00"
  130.         if not isdir(d):
  131.             os.makedirs(d)
  132.  
  133.     # create the log files
  134.     t = cmdutil.changeset_templater(ui, repo, patch=False, diffopts=None, mapfile=None, buffered=False)
  135.     t.use_template("""<div style='float: right; padding-left: 0.5em'><em>({author|person})</em></div><strong> {date|shortdate}: <a href='../commit/{node}.html'>{desc|strip|fill68|firstline}</a> <span style='font-size: xx-small'>{branches} {tags}</span><p>{desc|escape}</p>""")
  136.     logs = []
  137.     for ck in range(0, len(repo.changelog)/100+1):
  138.         ui.pushbuffer()
  139.         if ck:
  140.             dd = d
  141.             di = str(ck)+"00"
  142.             d = commits+"-"+di
  143.             logs[-1].write("<p><a href=\"../commits-"+di+"\">earlier</a></p>")
  144.             if ck>2:
  145.                 # the older log gets a reference to the newer one
  146.                 logs[-1].write("<p><a href=\"../commits-"+str(ck-2)+"00"+"\">later</a></p>")
  147.             elif ck>1:
  148.                 logs[-1].write("<p><a href=\"../commits\">later</a></p>")
  149.             logs.append(open(join(d, "index.html"), "w"))
  150.         else:
  151.             d = commits
  152.             logs.append(open(join(d, "index.html"), "w"))
  153.  
  154.         logs[-1].write(templates["head"].replace("{reponame}", name))
  155.         for c in range(ck*100+1, min(len(repo.changelog), (ck+1)*100)):
  156.             ctx = repo.changectx(str(-c))
  157.             t.show(ctx)
  158.         logs[-1].write(ui.popbuffer())
  159.  
  160.     for l in logs:
  161.         l.write(templates["foot"].replace("{reponame}", name))
  162.         l.close()
  163.  
  164.  
  165. def writecommits(ui, repo, target, name, force=False):
  166.     """Write all not yet existing commit files."""
  167.     commit = join(target, "commit")
  168.  
  169.     # create the folders
  170.     if not isdir(commit):
  171.         os.makedirs(commit)
  172.  
  173.     t = cmdutil.changeset_templater(ui, repo, patch=False, diffopts=None, mapfile=None, buffered=False)
  174.     t.use_template("""<div style='float: right; padding-left: 0.5em'><em>({author|person})</em></div><strong> {date|shortdate}: <a href='../commit/{node}.html'>{desc|strip|fill68|firstline}</a> <span style='font-size: xx-small'>{branches} {tags}</span><p>{desc|escape}</p>""")
  175.     for c in range(len(repo.changelog)):
  176.         ctx = repo.changectx(str(c))
  177.         cpath = join(commit, ctx.hex() + ".html")
  178.         if not force and isfile(cpath):
  179.             continue
  180.         with open(cpath, "w") as cf:
  181.             cf.write(templates["head"].replace("{reponame}", name))
  182.             ui.pushbuffer()
  183.             t.show(ctx)
  184.             cf.write(ui.popbuffer())
  185.             ui.pushbuffer()
  186.             commands.diff(ui, repo, change=str(c), git=True)
  187.             cf.write("<pre>"+ui.popbuffer().replace("<", "<")+"</pre>")
  188.             cf.write(templates["foot"].replace("{reponame}", name))
  189.  
  190.  
  191. def escapename(filename):
  192.     """escape index.html as .index.html and .ind… as ..ind… and so fort."""
  193.     if _indexregexp.match(filename) is not None:
  194.         return "." + filename
  195.     else: return filename
  196.  
  197.  
  198. def parsesrcdata(data):
  199.     """Parse a src file into a html file."""
  200.     return "<pre>"+data.replace("<", "<")+"</pre>"
  201.  
  202. def srcpath(target, ctx, filename):
  203.     """Get the relative path to the static sourcefile for an already escaped filename."""
  204.     return join(target,"src",ctx.hex(),filename+".html")
  205.  
  206. def rawpath(target, ctx, filename):
  207.     """Get the relative path to the static sourcefile for an already escaped filename."""
  208.     return join(target,"raw",ctx.hex(),filename)
  209.  
  210. def createindex(target, ctx):
  211.     """Create an index page for the changecontext: the commit message + the user + all files in the changecontext."""
  212.     # first the head
  213.     index = templates["manifesthead"].replace(
  214.         "{hex}", ctx.hex()).replace(
  215.         "{desc}", ctx.description()).replace(
  216.             "{user}", ctx.user())
  217.     # then the files
  218.     index += "<ul>"
  219.     for filename in ctx:
  220.         filectx = ctx[filename]
  221.         lasteditctx = filectx.filectx(filectx.filerev())
  222.         index += "<li><a href='../../../"+ srcpath(target, lasteditctx, escapename(filename)) + "'>" + filename + "</a> (<a href='../../../" + rawpath(target, lasteditctx, filename) + "'>raw</a>)</li>"
  223.     index += "</ul>"
  224.     return index
  225.  
  226. def writesourcetree(ui, repo, target, name, force):
  227.     """Write manifests for all commits and websites for all files.
  228.  
  229.     * For each file, write sites for all revisions where the file was changed: under src/<hex>/path as html site (with linenumbers and maybe colored source), under raw/<hex>/<path> as plain files. If there is an index.html file, write it as .index.html. If there also is .index.html, turn it to ..index.html, …
  230.     * For each commit write an index with links to the included files at their latest revisions before/at the commit.
  231.     """
  232.     # first write all files in all commits.
  233.     for c in range(len(repo.changelog)):
  234.         ctx = repo.changectx(str(c))
  235.         for filename in ctx.files():
  236.             filectx = ctx[filename]
  237.             # first write the raw data
  238.             filepath = rawpath(target,ctx,filectx.path())
  239.             # skip already existing files
  240.             if not force and isfile(filepath):
  241.                 continue
  242.             try:
  243.                 os.makedirs(dirname(filepath))
  244.             except OSError: pass # exists
  245.             with open(filepath, "w") as f:
  246.                 f.write(filectx.data())
  247.             # then write it as html
  248.             _filenameescaped = escapename(filectx.path())
  249.             filepath = srcpath(target,ctx,_filenameescaped)
  250.             try:
  251.                 os.makedirs(dirname(filepath))
  252.             except OSError: pass # exists
  253.             with open(filepath, "w") as f:
  254.                 f.write(templates["head"].replace("{reponame}", name))
  255.                 f.write(parsesrcdata(filectx.data()))
  256.                 f.write(templates["foot"].replace("{reponame}", name))
  257.     # then write manifests for all commits
  258.     for c in range(len(repo.changelog)):
  259.         ctx = repo.changectx(str(c))
  260.         filepath = join(target,"src",ctx.hex(),"index.html")
  261.         # skip already existing files
  262.         if not force and isfile(filepath):
  263.             continue
  264.         try:
  265.             os.makedirs(dirname(filepath))
  266.         except OSError: pass # exists
  267.         with open(filepath, "w") as f:
  268.             f.write(templates["head"].replace("{reponame}", name))
  269.             f.write(createindex(target, ctx))
  270.             f.write(templates["foot"].replace("{reponame}", name))
  271.  
  272.  
  273. def parsesite(ui, repo, target, **opts):
  274.     """Create the static folder."""
  275.     idfile = join(target, _staticidentifier)
  276.     if not isdir(target):
  277.         # make sure the target exists
  278.         os.makedirs(target)
  279.     else: # make sure it is a staticrepo
  280.         if not isfile(idfile):
  281.             if not ui.prompt("The target folder exists is no static repo. Really use it?", default="n").lower() in ["y", "yes"]:
  282.                 return
  283.     with open(idfile, "w") as i:
  284.         i.write("")
  285.  
  286.     if opts["name"]:
  287.         name = opts["name"]
  288.     elif target != "static": name = target
  289.     else: name = basename(repo.root)
  290.  
  291.     # first the stylesheets
  292.     screenstyle = opts["screenstyle"]
  293.     if screenstyle:
  294.         shutil.copyfile(screenstyle, join(target, "style.css"))
  295.     else:
  296.         with open(join(target, "style.css"), "w") as f:
  297.             f.write(templates["screenstyle"])
  298.     printstyle = opts["printstyle"]
  299.     if printstyle:
  300.         shutil.copyfile(printstyle, join(target, "print.css"))
  301.     else:
  302.         with open(join(target, "print.css"), "w") as f:
  303.             f.write(templates["printstyle"])
  304.  
  305.     # then the overview
  306.     writeoverview(ui, repo, target, name)
  307.  
  308.     # and the log
  309.     writelog(ui, repo, target, name)
  310.  
  311.     # and all commit files
  312.     writecommits(ui, repo, target, name, force=opts["force"])
  313.  
  314.     # and all file data
  315.     writesourcetree(ui, repo, target, name, force=opts["force"])
  316.  
  317.  
  318. def addrepo(ui, repo, target):
  319.     """Add the repo to the target and make sure it is up to date."""
  320.     try:
  321.         commands.init(ui, dest=target)
  322.     except mercurial.error.RepoError, e:
  323.         # already exists
  324.         pass
  325.     ui.pushbuffer()
  326.     commands.push(ui, repo, dest=target)
  327.     ui.popbuffer()
  328.  
  329.  
  330. def upload(ui, repo, target, ftpstring, force):
  331.     """upload the repo to the FTP server identified by the ftp string."""
  332.     user, password = ftpstring.split("@")[0].split(":")
  333.     serverandpath = "@".join(ftpstring.split("@")[1:])
  334.     server = serverandpath.split("/")[0]
  335.     ftppath = "/".join(serverandpath.split("/")[1:])
  336.     timeout = 10
  337.     try:
  338.         ftp = ftplib.FTP(server, user, password, "", timeout)
  339.     except socket.timeout:
  340.         ui.warn("connection to ", server, " timed out after ", timeout, " seconds.\n")
  341.         return
  342.  
  343.     ui.status(ftp.getwelcome(), "\n")
  344.  
  345.     # create the target dir.
  346.     serverdir = dirname(ftppath)
  347.     serverdirparts = ftppath.split("/")
  348.     sd = serverdirparts[0]
  349.     if not sd in ftp.nlst():
  350.         ftp.mkd(sd)
  351.     for sdp in serverdirparts[1:]:
  352.         sdo = sd
  353.         sd = os.path.join(sd, sdp)
  354.         if not sd in ftp.nlst(sdo):
  355.             ftp.mkd(sd)
  356.  
  357.  
  358.     ftp.cwd(ftppath)
  359.     if not ftp.pwd() == "/" + ftppath:
  360.         ui.warn("not in the correct ftp directory. Cowardly bailing out.\n")
  361.         return
  362.  
  363.     #ftp.dir()
  364.     #return
  365.     ftpfeatures = ftp.sendcmd("FEAT")
  366.  
  367.     _ftpdircache = set()
  368.  
  369.     for d, dirnames, filenames in os.walk(target):
  370.         for filename in filenames:
  371.             localfile = join(d, filename)
  372.             serverfile = localfile[len(target)+1:]
  373.             serverdir = dirname(serverfile)
  374.             serverdirparts = serverdir.split("/")
  375. #            print serverdirparts, serverfile
  376.             with open(localfile, "rb") as f:
  377.                 sd = serverdirparts[0]
  378.                 if sd and not sd in _ftpdircache and not sd in ftp.nlst():
  379.                     try:
  380.                         ui.status("creating directory ", sd, "\n")
  381.                         ftp.mkd(sd)
  382.                         _ftpdircache.add(sd)
  383.                     except ftplib.error_perm, resp:
  384.                         ui.warn("could not create directory ", sd, ": " , resp, "\n")
  385.                     else: _ftpdircache.add(sd)
  386.  
  387.                 for sdp in serverdirparts[1:]:
  388.                     sdold = sd
  389.                     sd = join(sd, sdp)
  390.                     #print sd, sdp
  391.                     #print ftp.nlst(sdold)
  392.                     if sd and not sd in _ftpdircache and not sd in ftp.nlst(sdold):
  393.                         try:
  394.                             ui.status("creating directory ", sd, "\n")
  395.                             ftp.mkd(sd)
  396.                             _ftpdircache.add(sd)
  397.                         except ftplib.error_perm, resp:
  398.                             ui.warn("could not create directory ", sd, ": " , resp, "\n")
  399.                     else: _ftpdircache.add(sd)
  400.  
  401.  
  402.                 if not serverfile in ftp.nlst(serverdir) or force:
  403.                     ui.status("uploading ", serverfile, " because it is not yet online.\n")
  404.                     ftp.storbinary("STOR "+ serverfile, f)
  405.                 else:
  406.                     # reupload the file if the file on the server is older than the local file.
  407.                     if " MDTM" in ftpfeatures.splitlines():
  408.                         ftpmtime = ftp.sendcmd("MDTM " + serverfile).split()[1]
  409.                         localmtime = os.stat(localfile).st_mtime
  410.                         localmtimestr = datetime.datetime.utcfromtimestamp(localmtime).strftime("%Y%m%d%H%M%S")
  411.                         newer = int(localmtimestr) > int(ftpmtime)
  412.                         if newer:
  413.                             ui.status("uploading ", serverfile, " because it is newer than the file on the FTP server.\n")
  414.                             ftp.storbinary("STOR "+ serverfile, f)
  415.  
  416.  
  417.  
  418. def static(ui, repo, target=None, **opts):
  419.     """Create a static copy of the repository and/or upload it to an FTP server."""
  420.     if repo.root == target:
  421.         ui.warn("static target repo can’t be the current repo")
  422.         return
  423.     # first: just create the site.
  424.     if not target: target = "static"
  425.     parsesite(ui, repo, target, **opts)
  426.     # add the hg repo to the static site
  427.     addrepo(ui, repo, target)
  428.     if opts["upload"]:
  429.         # upload the repo
  430.         upload(ui, repo, target, opts["upload"], opts["force"])
  431.  
  432.  
  433.  
  434. cmdtable = {
  435.     # "command-name": (function-call, options-list, help-string)
  436.     "static": (static,
  437.                      [
  438.                       #('r', 'rev', None, 'parse the given revision'),
  439.                       #('a', 'all', None, 'parse all revisions (requires much space)'),
  440.                       ('n', 'name', "", 'the repo name. Default: folder or last segment of the repo-path.'),
  441.                       ('u', 'upload', "", 'upload the repo to the given ftp host. Format: user:password@host/path/to/dir'),
  442.                       ('f', 'force', False, 'force recreating all commit files. Slow.'),
  443.                       ('s', 'screenstyle', "", 'use a custom stylesheet for display on screen'),
  444.                       ('p', 'printstyle', "", 'use a custom stylesheet for printing')],
  445.                      "[options] [folder]")
  446. }