DS Adapter for REST endpoint requiring pagnation

Ember Data
1

In this case I have zero control over the REST endpoint for retrieving data. Hence, I am taking a stab at creating an adaptor. The challenge arises with the findAll method, since the maximum number of features exceeds the maximum number of features the REST endpoint will return. However, the REST endpoint does support pagnation. As a result, I am trying to sort out how to navigate the world of promises to get all the records returned…and apparently not doing a very good job. Here is the code I have thus far. Thank you in advance for any assistance or insight you may be able to provide.

./app/adapters/reach.js

import DS from 'ember-data';`
import request from '../utils/request';
import ENV from '../config/environment';
import Ember from 'ember';

const urlFeatureLayer = ENV.APP.ARCGIS.POINTS.URL;
const urlQuery = `${urlFeatureLayer}/query`;
const uidField = 'reachId';  // not the same one ArcGIS uses

// for quickly making a get request to the ArcGIS REST endpoint
const makeQueryRequest = (opts) => {
  opts.f = 'json';  // just ensuring this is taken care of

  // providing a default if none is provided
  if (!opts.where){
    opts.where = '1=1';
  }

  // using Ember's built in jQuery to parameterize inputs for request
  opts = Ember.$.param(opts);

  // combine params onto url, and make request
  let url = `${urlQuery}?${opts}`;
  return request(url, {method: 'GET'});
}

// pulling together the steps to side step the maximum record response limit
let retrieveAll = () => {

  // retrieve the maximum feature count
  request(`${urlFeatureLayer}?f=json`, {method: 'GET'})
  .then((respProp) => {
    Ember.debug(`Max Record Count: ${respProp.maxRecordCount}`);

    // get the feature count
    makeQueryRequest({returnCountOnly: true})
    .then((respCnt) => {
      Ember.debug(`Feature Count: ${respCnt.count}`);

      // calculate the number of pulls required to get all the data
      let pullCount = Math.ceil(respCnt.count / respProp.maxRecordCount);
      Ember.debug(`Pull Count: ${pullCount}`);

      // iteratively, in chunks, retrieve all the features
      let allFeatures = [];
      for (let i = 0; i < pullCount; i++){
        let offset = i * respProp.maxRecordCount;
        let recordCount = respProp.maxRecordCount;
        allFeatures.push(makeQueryRequest({
          resultOffset: offset,
          resultRecordCount: recordCount
        }));
      }

      // catcher to get all the promises and send back the results
      Promise.all(allFeatures).then((allFeatures) => {
        return allFeatures;
      });

    });

  });

}

export default DS.JSONAPIAdapter.extend({

  findAll: function(store) {

    return retrieveAll();

    // let opts = Ember.$.param({
    //   where: '1=1',
    //   f: 'json',
    //   outFields: '*'
    // });
    // let url = `${urlQuery}?${opts}`;
    // return request(url, {method: 'GET'});
  },

  findRecord (store, type, id) {
    let opts = Ember.$.param({
      where: `${uidField} = ${id}`,
      f: 'json',
      outFields: '*'
    });
    let url = `${urlQuery}?${opts}`;
    return request(url, {method: 'GET'});
  }

});

The referenced request is really little more than a wrapper for fetch, but here it is for reference…

./utils/request.js

import Ember from 'ember';
import fetch from 'fetch';
import addToken from './add-token';
import encodeForm from './encode-form';

/**
 * Fetch based request method
 */
export default function request (url, opts = {}) {

  // if we are POSTing, we need to manually set the content-type because AGO
  // actually does care about this header
  if (opts.method && opts.method === 'POST') {
    if (!opts.headers) {
      opts.headers = {
        'Accept': 'application/json, application/xml, multipart/form-data, text/plain, text/html, *.*',
        'Content-Type': 'multipart/form-data'
      };
    }

    // if we have a data, prep it to send
    if (opts.data) {
      opts.body = encodeForm(opts.data);
    }
  }

  opts.redirect = 'follow';
  opts.mode = 'cors';

  // add a token if provided
  url = addToken(url, opts.token);

  Ember.debug('Making request to ' + url);

  return fetch(url, opts).then(checkStatusAndParseJson);
}

/**
 * Fetch does not reject on non-200 responses, so we need to check this manually
 */
function checkStatusAndParseJson (response) {
  let error;
  Ember.debug('Fetch request status: ' + response.status);

  // check if this is one of those groovy 200-but-a-400 things
  if (response.status >= 200 && response.status < 300) {
    return response.json().then(json => {

      // cook an error
      if (json.error) {
        error = new Error(json.error.message);
        error.code = json.error.code || 404;
        error.response = response;
        Ember.debug('Error in response:  ' + json.error.message);
        throw error;

      } else {
        return json;
      }
    });

  } else {
    // Response has non 200 http code
    error = new Error(response.statusText);
    error.response = response;
    throw error;
  }
}
2

There are use cases where a paginated end point is a pain–such as when you want to do your own aggregation. But normally I would embrace the paginated API instead of bending it to my will. It is most likely a restriction for a reason. If there isn’t an “all” option on the request, I would fold pagination into my UI somehow. We currently use an infinite scroll.

3

@Chris_Lincoln, I took your advice, implementing a search field producing a filtered list display after at least three characters are entered. Still though, I am kind of curious as to how this is possible to overcome.

4

If the REST service supports a page size parameter and you can ascertain the maximum then send that as a parameter.