Merge pull request #79 from eeditiones/fix/betmas-collections

Make tuttle work with very active repos that are deployed into a subcollection

Author: line-o

src/index.html

@@ -89,7 +89,7 @@ <h1><img class="tuttle" src="images/HPTuttle-1866.png"></img>
  <fx-submission id="fetch"
  method="get"
  replace="none"
- url="./git/{instance('default')/repos/repo[index('list')]/@collection}"
+ url="./git/{instance('default')/repos/repo[index('list')]/@collection => encode-for-uri()}"
  serialization="none">
 
  <fx-action event="submit">
@@ -109,7 +109,7 @@ <h1><img class="tuttle" src="images/HPTuttle-1866.png"></img>
  method="post"
  replace="instance"
  instance="response"
- url="./git/{instance('default')/repos/repo[index('list')]/@collection}"
+ url="./git/{instance('default')/repos/repo[index('list')]/@collection => encode-for-uri()}"
  serialization="none">
 
  <fx-action event="submit-done">
@@ -122,7 +122,7 @@ <h1><img class="tuttle" src="images/HPTuttle-1866.png"></img>
  </fx-submission>
 
  <fx-submission id="update"
- url="./git/{instance('default')/repos/repo[index('list')]/@collection}/incremental"
+ url="./git/{instance('default')/repos/repo[index('list')]/@collection => encode-for-uri()}/incremental"
  replace="instance"
  instance="response"
  method="post">
@@ -138,7 +138,7 @@ <h1><img class="tuttle" src="images/HPTuttle-1866.png"></img>
  <fx-send submission="config"></fx-send>
  </fx-action>
  <fx-action event="submit-error">
- <fx-message>Udpate Failed!</fx-message>
+ <fx-message>Update Failed!</fx-message>
  <fx-setvalue ref="instance('vars')/inprogress"></fx-setvalue>
  </fx-action>
  </fx-submission>
@@ -213,4 +213,4 @@ <h3>Git Repositories</h3>
  <script type="module" src="js/fore-all.js"></script>
 </div>
 </body>
-</html>
+</html>

src/modules/api.xq

@@ -551,7 +551,7 @@ declare %private function api:get-collection-config($collection as xs:string?) a
  if (empty($git-collection))
  then error((), "git collection not found!")
  else if (empty($collection-config))
- then error((), "collection config not found!")
+ then error((), "collection config " || $git-collection || " not found!")
  else $collection-config
 };
 

src/modules/github.xqm

@@ -9,6 +9,9 @@ import module namespace config="http://e-editiones.org/tuttle/config" at "config
 
 declare namespace http="http://expath.org/ns/http-client";
 
+declare variable $github:max-page-size := 100;
+declare variable $github:max-total-result-size := 500;
+
 declare function github:repo-url($config as map(*)) as xs:string {
  ``[`{$config?baseurl}`repos/`{$config?owner}`/`{$config?repo}`]``
 };
@@ -74,7 +77,7 @@ declare function github:get-last-commit($config as map(*)) as map(*) {
  : Get all commits
  :)
 declare function github:get-commits($config as map(*)) as array(*)* {
- github:get-commits($config, 100)
+ github:get-commits($config, $github:max-page-size)
 };
 
 (:~
@@ -107,23 +110,50 @@ declare %private function github:short-commit-info ($commit-info as map(*)) as a
  : Get commits in full
  :)
 declare function github:get-raw-commits($config as map(*), $count as xs:integer) as array(*)? {
- github:request-json-ignore-pages(
- github:commit-ref-url($config, $count), $config?token)
+ github:get-raw-commits($config, $count, ())
+};
+
+(:~
+ : Get commits in full, going over pages until we find the commit with the correct hash
+ :)
+declare function github:get-raw-commits (
+ $config as map(*),
+ $count as xs:integer,
+ $stop-at-commit-id as xs:string?
+) as array(*)? {
+ let $stop-condition := if (empty($stop-at-commit-id)) then
+ function ($_) {
+ (: We are not looking for any SHA. Prevent going over all of the commits in a possibly big repo :)
+ true()
+ }
+ else
+ function ($results-on-page) {
+(: We can stop searching once we have all the commits between 'now' and the commit we looked for :)
+ let $found-commits := $results-on-page?*?sha
+ return $found-commits = $stop-at-commit-id
+ }
+
+ let $results := github:request-json-all-pages(
+ github:commit-ref-url($config, $count),
+ $config?token,
+ $stop-condition
+ )
+ return array { $results?* }
 };
 
 (:~
  : Get diff between production collection and github-newest
  :)
 declare function github:get-newest-commits($config as map(*)) as xs:string* {
  let $deployed := $config?deployed
- let $commits := github:get-raw-commits($config, 100)
+ let $commits := github:get-raw-commits($config, $github:max-page-size, $deployed)
  let $sha := $commits?*?sha
  let $how-many := index-of($sha, $deployed) - 1
  return
  if (empty($how-many)) then (
  error(
  xs:QName("github:commit-not-found"),
- 'The deployed commit hash ' || $deployed || ' was not found in the list of commits on the remote.')
+ 'The deployed commit hash ' || $deployed || ' was not found in the list of commits on the remote. Tuttle can only process incremental upgrades of ' || $github:max-total-result-size || '.')
  ) else (
  reverse(subsequence($sha, 1, $how-many))
  )
@@ -218,7 +248,7 @@ declare function github:incremental($config as map(*)) {
  :)
 declare function github:get-commit-files($config as map(*), $sha as xs:string) as array(*) {
  let $url := github:repo-url($config) || "/commits/" || $sha
- let $commit := github:request-json-all-pages($url, $config?token, ())
+ let $commit := github:request-json-all-pages($url, $config?token)
 
  return array { $commit?files?* }
 };
@@ -361,7 +391,32 @@ declare %private function github:request-json($url as xs:string, $token as xs:st
  )
 };
 
-declare %private function github:request-json-all-pages($url as xs:string, $token as xs:string?, $acc) {
+(:~
+ : Get all pages of a specified URL. Github has some paginated endpoints, This function traverses all of
+ : those and joins the results.
+ :)
+declare %private function github:request-json-all-pages($url as xs:string, $token as xs:string?) {
+ github:request-json-all-pages($url, $token, function ($_) { (: Traverse all pages :) false() }, ())
+};
+
+(:~
+ : Overload, adds the $stop-condition callback which is given the contents of the current page
+ : return `true()` to indicate there are sufficient results and we can stop
+ :)
+declare %private function github:request-json-all-pages(
+ $url as xs:string,
+ $token as xs:string?,
+ $stop-condition as function(map(*)) as xs:boolean
+) {
+ github:request-json-all-pages($url, $token, $stop-condition, ())
+};
+
+declare %private function github:request-json-all-pages(
+ $url as xs:string,
+ $token as xs:string?,
+ $stop-condition as function(map(*)) as xs:boolean,
+ $acc
+) {
  let $response :=
  app:request-json(
  github:build-request($url, (
@@ -374,13 +429,16 @@ declare %private function github:request-json-all-pages($url as xs:string, $toke
  github:parse-link-header($response[1]/http:header[@name="link"]/@value)?next
  ) else ()
 
- let $all := ($acc, $response[2])
+ let $results-in-this-page := $response[2]
+ let $all := ($acc, $results-in-this-page)
+
+ let $should-stop := $stop-condition($results-in-this-page)
 
  return (
- if (exists($next-url)) then (
-  github:request-json-all-pages($next-url, $token, $all)
+ if (not($should-stop) and count($all?*) < $github:max-total-result-size and exists($next-url)) then (
+ github:request-json-all-pages($next-url, $token, $stop-condition, $all)
  ) else (
-  $all
+ $all
  )
  )
 };

test/fixtures/alt-big-repo-tuttle.xml

@@ -0,0 +1,31 @@
+<tuttle>
+ <repos>
+ <!-- A version of tuttle sample data with tons of commits so updates go over multiple pages -->
+ <collection name="tuttle-sample-data">
+ <default>true</default>
+ <type>github</type>
+ <baseurl>https://api.github.com/</baseurl>
+ <repo>tuttle-sample-data</repo>
+ <owner>eeditiones</owner>
+ <token>XXX</token>
+ <ref>long-history</ref>
+ <hookuser>admin</hookuser>
+ <hookpasswd />
+ </collection>
+ </repos>
+ <ignore>
+ <file>existdb.json</file>
+ <file>build.xml</file>
+ <file>README.md</file>
+ <file>.gitignore</file>
+ <file>expath-pkg.xml.tmpl</file>
+ <file>repo.xml.tmpl</file>
+ <file>build.properties.xml</file>
+ </ignore>
+ <config
+ apikeys="/db/system/auth/tuttle-token.xml"
+ lock="git-lock.xml"
+ prefix="/db/apps/"
+ suffix="-stage"
+ ><sm group="nogroup" mode="rw-r--r--" user="nobody" /></config>
+</tuttle>

test/fixtures/alt-tuttle.xml

@@ -36,4 +36,4 @@
  <config prefix="/db/apps/" suffix="-stage" lock="git-lock.xml" apikeys="/db/system/auth/tuttle-token.xml">
  <sm user="nobody" group="nogroup" mode="rw-r--r--"/>
  </config>
-</tuttle>
+</tuttle>

test/tuttle.js

@@ -110,7 +110,7 @@ export default () =>
  await assert.doesNotReject(resultPromise, 'The request should succeed');
  res = await resultPromise;
  })
- 
+
  it('returns status 200', async function () {
  assert.strictEqual(res.status, 200);
  });
@@ -176,27 +176,62 @@ export default () =>
  it('can also write hashes to repo.xml', async () => {
  await remove();
  await install();
- 
+
  // Set up tuttle with a repo where repo.xml is used to store the git sha info
  const buffer = await readFile('./test/fixtures/alt-repoxml-tuttle.xml');
  await putResource(buffer, '/db/apps/tuttle/data/tuttle.xml');
- 
+
  const resultPromise = axios.get('git/status', { auth });
  await assert.doesNotReject(resultPromise);
- 
+
  const stagingPromise = axios.get('git/tuttle-sample-data', { auth });
  await assert.doesNotReject(stagingPromise, 'The request should succeed');
 
  const deployPromise = axios.post('git/tuttle-sample-data', {}, { auth });
  await assert.doesNotReject(deployPromise, 'The request should succeed');
- 
+
  const repoXML = await getResource('/db/apps/tuttle-sample-data/repo.xml');
- 
+
  const repo = new DOMParser().parseFromString(repoXML.toString(), 'text/xml').documentElement;
  assert.ok(repo.getAttribute('commit-id'), 'The commit id should be set');
  assert.ok(repo.getAttribute('commit-time'), 'The commit time should be set');
  assert.ok(repo.getAttribute('commit-date'), 'The commit date should be set');
  });
- 
- });
 
+ describe('large histories', async () => {
+ before(async () => {
+ await remove();
+ await install();
+ });
+
+ await it('can upgrade over a few hundred commits', async () => {
+ // Set up tuttle with a repo with a ton of commits that it can upgrade over
+ const buffer = await readFile('./test/fixtures/alt-big-repo-tuttle.xml');
+ await putResource(buffer, '/db/apps/tuttle/data/tuttle.xml');
+
+ const OLD_HASH = '41188098f120b6e70d1b0c3bb704a422eba43dfa';
+ const stageOldVersionPromise = axios.get(`git/tuttle-sample-data?hash=${OLD_HASH}`, { auth });
+ await assert.doesNotReject(stageOldVersionPromise);
+ const deployOldVersionPromise = axios.post('git/tuttle-sample-data', {}, { auth });
+ await assert.doesNotReject(deployOldVersionPromise, 'The request should succeed');
+
+ const beforeString = await getResource('/db/apps/tuttle-sample-data/data/regular-changing-document.xml');
+
+ const before = new DOMParser().parseFromString(beforeString.toString(), 'text/xml').documentElement;
+ assert.strictEqual(before.textContent, 'Initial version');
+
+ console.log('deployed older version of the sample data on the long-history branch')
+
+ const resultPromise = axios.get('git/status', { auth });
+ await assert.doesNotReject(resultPromise);
+
+ const incrementalPromise = axios.post('git/tuttle-sample-data/incremental', {}, { auth });
+ await assert.doesNotReject(incrementalPromise, 'The incremental request should succeed');
+
+ const afterString = await getResource('/db/apps/tuttle-sample-data/data/regular-changing-document.xml');
+
+ const after = new DOMParser().parseFromString(afterString.toString(), 'text/xml').documentElement;
+ assert.strictEqual(after.textContent, 'change for 200');
+ });
+ });
+ });