dojo.provide("dojox.data.CsvStore"); dojo.require("dojo.data.util.filter"); dojo.require("dojo.data.util.simpleFetch"); dojo.declare("dojox.data.CsvStore", null, { // summary: // The CsvStore implements the dojo.data.api.Read API and reads // data from files in CSV (Comma Separated Values) format. // All values are simple string values. References to other items // are not supported as attribute values in this datastore. // // Example data file: // name, color, age, tagline // Kermit, green, 12, "Hi, I'm Kermit the Frog." // Fozzie Bear, orange, 10, "Wakka Wakka Wakka!" // Miss Piggy, pink, 11, "Kermie!" // // Note that values containing a comma must be enclosed with quotes ("") // Also note that values containing quotes must be escaped with two consecutive quotes (""quoted"") /* examples: * var csvStore = new dojox.data.CsvStore({url:"movies.csv"); * var csvStore = new dojox.data.CsvStore({url:"http://example.com/movies.csv"); */ constructor: function(/* Object */ keywordParameters){ // summary: initializer // keywordParameters: {url: String} // keywordParameters: {data: String} // keywordParameters: {label: String} The column label for the column to use for the label returned by getLabel. // keywordParameters: {identifier: String} The column label for the column to use for the identity. Optional. If not set, the identity is the row number. this._attributes = []; // e.g. ["Title", "Year", "Producer"] this._attributeIndexes = {}; // e.g. {Title: 0, Year: 1, Producer: 2} this._dataArray = []; // e.g. [[],[],[]] this._arrayOfAllItems = []; // e.g. [{_csvId:0,_csvStore:store},...] this._loadFinished = false; if(keywordParameters.url){ this.url = keywordParameters.url; } this._csvData = keywordParameters.data; if(keywordParameters.label){ this.label = keywordParameters.label; }else if(this.label === ""){ this.label = undefined; } this._storeProp = "_csvStore"; // Property name for the store reference on every item. this._idProp = "_csvId"; // Property name for the Item Id on every item. this._features = { 'dojo.data.api.Read': true, 'dojo.data.api.Identity': true }; this._loadInProgress = false; //Got to track the initial load to prevent duelling loads of the dataset. this._queuedFetches = []; this.identifier = keywordParameters.identifier; if (this.identifier === "") { delete this.identifier; }else{ this._idMap = {}; } }, url: "", //Declarative hook for setting Csv source url. label: "", //Declarative hook for setting the label attribute. identifier: "", //Declarative hook for setting the identifier. _assertIsItem: function(/* item */ item){ // summary: // This function tests whether the item passed in is indeed an item in the store. // item: // The item to test for being contained by the store. if(!this.isItem(item)){ throw new Error(this.declaredClass + ": a function was passed an item argument that was not an item"); } }, _assertIsAttribute: function(/* item || String */ attribute){ // summary: // This function tests whether the item passed in is indeed a valid 'attribute' like type for the store. // attribute: // The attribute to test for being contained by the store. if(!dojo.isString(attribute)){ throw new Error(this.declaredClass + ": a function was passed an attribute argument that was not an attribute object nor an attribute name string"); } }, _getIndex: function(item){ // summary: // Internal function to get the internal index to the item data from the item handle // item: // The idem handle to get the index for. var idx = this.getIdentity(item); if(this.identifier){ idx = this._idMap[idx]; } return idx; }, /*************************************** dojo.data.api.Read API ***************************************/ getValue: function( /* item */ item, /* attribute || attribute-name-string */ attribute, /* value? */ defaultValue){ // summary: // See dojo.data.api.Read.getValue() // Note that for the CsvStore, an empty string value is the same as no value, // so the defaultValue would be returned instead of an empty string. this._assertIsItem(item); this._assertIsAttribute(attribute); var itemValue = defaultValue; if(this.hasAttribute(item, attribute)){ var itemData = this._dataArray[this._getIndex(item)]; itemValue = itemData[this._attributeIndexes[attribute]]; } return itemValue; //String }, getValues: function(/* item */ item, /* attribute || attribute-name-string */ attribute){ // summary: // See dojo.data.api.Read.getValues() // CSV syntax does not support multi-valued attributes, so this is just a // wrapper function for getValue(). var value = this.getValue(item, attribute); return (value ? [value] : []); //Array }, getAttributes: function(/* item */ item){ // summary: // See dojo.data.api.Read.getAttributes() this._assertIsItem(item); var attributes = []; var itemData = this._dataArray[this._getIndex(item)]; for(var i=0; i= 0 && identity < this._dataArray.length){ return true; //Boolean } } } return false; //Boolean }, isItemLoaded: function(/* anything */ something){ // summary: // See dojo.data.api.Read.isItemLoaded() // The CsvStore always loads all items, so if it's an item, then it's loaded. return this.isItem(something); //Boolean }, loadItem: function(/* item */ item){ // summary: // See dojo.data.api.Read.loadItem() // description: // The CsvStore always loads all items, so if it's an item, then it's loaded. // From the dojo.data.api.Read.loadItem docs: // If a call to isItemLoaded() returns true before loadItem() is even called, // then loadItem() need not do any work at all and will not even invoke // the callback handlers. }, getFeatures: function(){ // summary: // See dojo.data.api.Read.getFeatures() return this._features; //Object }, getLabel: function(/* item */ item){ // summary: // See dojo.data.api.Read.getLabel() if(this.label && this.isItem(item)){ return this.getValue(item,this.label); //String } return undefined; //undefined }, getLabelAttributes: function(/* item */ item){ // summary: // See dojo.data.api.Read.getLabelAttributes() if(this.label){ return [this.label]; //array } return null; //null }, // The dojo.data.api.Read.fetch() function is implemented as // a mixin from dojo.data.util.simpleFetch. // That mixin requires us to define _fetchItems(). _fetchItems: function( /* Object */ keywordArgs, /* Function */ findCallback, /* Function */ errorCallback){ // summary: // See dojo.data.util.simpleFetch.fetch() var self = this; var filter = function(requestArgs, arrayOfAllItems){ var items = null; if(requestArgs.query){ var key, value; items = []; var ignoreCase = requestArgs.queryOptions ? requestArgs.queryOptions.ignoreCase : false; //See if there are any string values that can be regexp parsed first to avoid multiple regexp gens on the //same value for each item examined. Much more efficient. var regexpList = {}; for(key in requestArgs.query){ value = requestArgs.query[key]; if(typeof value === "string"){ regexpList[key] = dojo.data.util.filter.patternToRegExp(value, ignoreCase); } } for(var i = 0; i < arrayOfAllItems.length; ++i){ var match = true; var candidateItem = arrayOfAllItems[i]; for(key in requestArgs.query){ value = requestArgs.query[key]; if(!self._containsValue(candidateItem, key, value, regexpList[key])){ match = false; } } if(match){ items.push(candidateItem); } } }else{ // We want a copy to pass back in case the parent wishes to sort the array. We shouldn't allow resort // of the internal list so that multiple callers can get lists and sort without affecting each other. if(arrayOfAllItems.length> 0){ items = arrayOfAllItems.slice(0,arrayOfAllItems.length); } } findCallback(items, requestArgs); }; if(this._loadFinished){ filter(keywordArgs, this._arrayOfAllItems); }else{ if(this.url !== ""){ //If fetches come in before the loading has finished, but while //a load is in progress, we have to defer the fetching to be //invoked in the callback. if(this._loadInProgress){ this._queuedFetches.push({args: keywordArgs, filter: filter}); }else{ this._loadInProgress = true; var getArgs = { url: self.url, handleAs: "text" }; var getHandler = dojo.xhrGet(getArgs); getHandler.addCallback(function(data){ try{ self._processData(data); filter(keywordArgs, self._arrayOfAllItems); self._handleQueuedFetches(); }catch(e){ errorCallback(e, keywordArgs); } }); getHandler.addErrback(function(error){ self._loadInProgress = false; if(errorCallback){ errorCallback(error, keywordArgs); }else{ throw error; } }); } }else if(this._csvData){ try{ this._processData(this._csvData); this._csvData = null; filter(keywordArgs, this._arrayOfAllItems); }catch(e){ errorCallback(e, keywordArgs); } }else{ var error = new Error(this.declaredClass + ": No CSV source data was provided as either URL or String data input."); if(errorCallback){ errorCallback(error, keywordArgs); }else{ throw error; } } } }, close: function(/*dojo.data.api.Request || keywordArgs || null */ request){ // summary: // See dojo.data.api.Read.close() }, // ------------------------------------------------------------------- // Private methods _getArrayOfArraysFromCsvFileContents: function(/* string */ csvFileContents){ /* summary: * Parses a string of CSV records into a nested array structure. * description: * Given a string containing CSV records, this method parses * the string and returns a data structure containing the parsed * content. The data structure we return is an array of length * R, where R is the number of rows (lines) in the CSV data. The * return array contains one sub-array for each CSV line, and each * sub-array contains C string values, where C is the number of * columns in the CSV data. */ /* example: * For example, given this CSV string as input: * "Title, Year, Producer \n Alien, 1979, Ridley Scott \n Blade Runner, 1982, Ridley Scott" * this._dataArray will be set to: * [["Alien", "1979", "Ridley Scott"], * ["Blade Runner", "1982", "Ridley Scott"]] * And this._attributes will be set to: * ["Title", "Year", "Producer"] * And this._attributeIndexes will be set to: * { "Title":0, "Year":1, "Producer":2 } */ if(dojo.isString(csvFileContents)){ var lineEndingCharacters = new RegExp("\r\n|\n|\r"); var leadingWhiteSpaceCharacters = new RegExp("^\\s+",'g'); var trailingWhiteSpaceCharacters = new RegExp("\\s+$",'g'); var doubleQuotes = new RegExp('""','g'); var arrayOfOutputRecords = []; var i; var arrayOfInputLines = this._splitLines(csvFileContents); for(i = 0; i < arrayOfInputLines.length; ++i){ var singleLine = arrayOfInputLines[i]; if(singleLine.length > 0){ var listOfFields = singleLine.split(','); var j = 0; while(j < listOfFields.length){ var space_field_space = listOfFields[j]; var field_space = space_field_space.replace(leadingWhiteSpaceCharacters, ''); // trim leading whitespace var field = field_space.replace(trailingWhiteSpaceCharacters, ''); // trim trailing whitespace var firstChar = field.charAt(0); var lastChar = field.charAt(field.length - 1); var secondToLastChar = field.charAt(field.length - 2); var thirdToLastChar = field.charAt(field.length - 3); if(field.length === 2 && field == "\"\""){ listOfFields[j] = ""; //Special case empty string field. }else if((firstChar == '"') && ((lastChar != '"') || ((lastChar == '"') && (secondToLastChar == '"') && (thirdToLastChar != '"')))){ if(j+1 === listOfFields.length){ // alert("The last field in record " + i + " is corrupted:\n" + field); return null; //null } var nextField = listOfFields[j+1]; listOfFields[j] = field_space + ',' + nextField; listOfFields.splice(j+1, 1); // delete element [j+1] from the list }else{ if((firstChar == '"') && (lastChar == '"')){ field = field.slice(1, (field.length - 1)); // trim the " characters off the ends field = field.replace(doubleQuotes, '"'); // replace "" with " } listOfFields[j] = field; j += 1; } } arrayOfOutputRecords.push(listOfFields); } } // The first item of the array must be the header row with attribute names. this._attributes = arrayOfOutputRecords.shift(); for(i = 0; i 0) { for(var i = 0; i < this._queuedFetches.length; i++){ var fData = this._queuedFetches[i]; var delayedFilter = fData.filter; var delayedQuery = fData.args; if(delayedFilter){ delayedFilter(delayedQuery, this._arrayOfAllItems); }else{ this.fetchItemByIdentity(fData.args); } } this._queuedFetches = []; } } }); //Mix in the simple fetch implementation to this class. dojo.extend(dojox.data.CsvStore,dojo.data.util.simpleFetch);