From 6ee96fb07c1357cdeee4df3d3bfc3b53d221d7a9 Mon Sep 17 00:00:00 2001 From: sedum Date: Thu, 19 Jan 2023 01:29:24 +0100 Subject: [PATCH] geolocation api rate-limit: improve the delay mechanism to be sure to don't hit provider more than 1 time/s, add memory-cache to save response data. --- package.json | 1 + server/api/controller/geolocation.js | 128 ++++++++++++++++++++++----- server/api/controller/place.js | 44 +-------- server/api/index.js | 4 +- yarn.lock | 5 ++ 5 files changed, 115 insertions(+), 67 deletions(-) diff --git a/package.json b/package.json index 437e4b93..7046e50a 100644 --- a/package.json +++ b/package.json @@ -62,6 +62,7 @@ "linkifyjs": "4.0.2", "lodash": "^4.17.21", "mariadb": "^3.0.1", + "memory-cache": "^0.2.0", "microformat-node": "^2.0.1", "minify-css-string": "^1.0.0", "mkdirp": "^1.0.4", diff --git a/server/api/controller/geolocation.js b/server/api/controller/geolocation.js index 9ff07ead..0233e919 100644 --- a/server/api/controller/geolocation.js +++ b/server/api/controller/geolocation.js @@ -1,9 +1,18 @@ const rateLimit = require('express-rate-limit'); const log = require('../../log') -let d // departure time +const NOMINATIM_URL = 'https://nominatim.openstreetmap.org/search' +const PHOTON_URL = 'https://photon.komoot.io/api/' +const axios = require('axios') +const { version } = require('../../../package.json') +const cache = require('memory-cache'); +let d = 0 // departure time +let h = 0 // hit geocoding provider time (aka Latency) const geolocationController = { - rateLimiter: rateLimit({ + /** + * TODO: replace/merge with a general 'instance rate-limiter' or 'instance api-related rate-limiter' when this will be defined + */ + instanceApiRateLimiter: rateLimit({ windowMs: 15 * 60 * 1000, // 15 minutes max: 100, // Limit each IP to 100 requests per `window` (here, per 15 minutes) standardHeaders: true, // Return rate limit info in the `RateLimit-*` headers @@ -14,33 +23,108 @@ const geolocationController = { * Limit api usage * From https://operations.osmfoundation.org/policies/nominatim/ * [Requirements] No heavy uses (an absolute maximum of 1 request per second). - * [Websites and Apps] Note that the usage limits above apply per website/application: the sum of traffic by all your users should not exceed the limits. + * [Websites and Apps] + * - Note that the usage limits above apply per website/application: the sum of traffic by all your users should not exceed the limits. + * - If at all possible, set up a proxy and also enable caching of requests. */ - apiLimit (req, res, next) { - let dprev = d // departure time of previous - let a = Date.now() // arrival time + providerRateLimit (req, res, next) { + let a = Date.now(); // arrival time + let dprev = d + d = dprev + 1000 + h - if (typeof dprev !== 'undefined') { - d = dprev + 1000 + console.log('a: ' + a) + console.log('dprev: ' + dprev) + console.log('d: ' + d) - if (a > d) { - d = a + 10 - geolocationController.rateLimiter(req, res, next) - } else { - let wait = d - a - log.warn('More than 1 request per second to geolocation api. This from ' + req.ip) - - setTimeout(() => { - geolocationController.rateLimiter(req, res, next) - }, wait) + // if the same request was already cached skip the delay mechanism + if (cache.get(req.params.place_details)) { + if (a < d) { + log.warn('More than 1 request per second to geolocation api. This from ' + req.ip + ' . The response data is served from memory-cache') } + // reset departure time because there is no need to ask provider + d = dprev + return next() + } + + if (d === 0 || a > d) { + // no-queue or old-queue + console.log('No queue or Old queue') + // arrival time + 10ms estimated computing time + d = a + 10 + next() } else { - d = a + 10 // add 10ms - geolocationController.rateLimiter(req, res, next) + // fresh queue + console.log('Fresh queue') + let wait = d - a + console.log('Waiting '+ wait) + log.warn('More than 1 request per second to geolocation api. This from ' + req.ip + ' . Applying ToS padding before asking to provider. The response data is now cached.') + + setTimeout(() => { + next() + }, wait) } - } - + }, + + async _nominatim (req, res) { + const details = req.params.place_details + const countrycodes = res.locals.settings.geocoding_countrycodes || [] + const geocoding_provider = res.locals.settings.geocoding_provider || NOMINATIM_URL + // ?limit=3&format=json&namedetails=1&addressdetails=1&q= + + const ret = await axios.get(`${geocoding_provider}`, { + params: { + countrycodes: countrycodes.join(','), + q: details, + limit: 3, + format: 'json', + addressdetails: 1, + namedetails: 1, + }, + headers: { 'User-Agent': `gancio ${version}` } + }) + + return res.json(ret.data) + + }, + + async _photon (req, res) { + const details = req.params.place_details + const geocoding_provider = res.locals.settings.geocoding_provider || PHOTON_URL + + if (cache.get(details)) { + console.log('Retrieving data from cache') + const ret = { + data: await cache.get(details) + } + return res.json(ret.data) + } else { + let RTTstart = Date.now() + + console.log('Asking Provider: ' + RTTstart) + const ret = await axios.get(`${geocoding_provider}`, { + params: { + q: details, + limit: 3, + }, + headers: { 'User-Agent': `gancio ${version}` } + }) + + if (ret) { + let RTTend = Date.now() + console.log('Response arrived: ' + RTTend) + // Save the hit time (aka Latency) + h = (RTTend - RTTstart) / 2 + console.log('Saving latency h: ' + h) + } + + console.log('Caching results') + cache.put(details, ret.data); + return res.json(ret.data) + } + + }, + } module.exports = geolocationController \ No newline at end of file diff --git a/server/api/controller/place.js b/server/api/controller/place.js index 8fd1aa3b..47f71b34 100644 --- a/server/api/controller/place.js +++ b/server/api/controller/place.js @@ -7,9 +7,6 @@ const { version } = require('../../../package.json') const log = require('../../log') const { Op, where, col, fn, cast } = require('sequelize') -const NOMINATIM_URL = 'https://nominatim.openstreetmap.org/search' -const PHOTON_URL = 'https://photon.komoot.io/api/' -const axios = require('axios') module.exports = { @@ -75,45 +72,6 @@ module.exports = { // TOFIX: don't know why limit does not work return res.json(places.slice(0, 10)) - }, - - async _nominatim (req, res) { - const details = req.params.place_details - const countrycodes = res.locals.settings.geocoding_countrycodes || [] - const geocoding_provider = res.locals.settings.geocoding_provider || NOMINATIM_URL - // ?limit=3&format=json&namedetails=1&addressdetails=1&q= - - const ret = await axios.get(`${geocoding_provider}`, { - params: { - countrycodes: countrycodes.join(','), - q: details, - limit: 3, - format: 'json', - addressdetails: 1, - namedetails: 1, - }, - headers: { 'User-Agent': `gancio ${version}` } - }) - - return res.json(ret.data) - - }, - - async _photon (req, res) { - const details = req.params.place_details - const geocoding_provider = res.locals.settings.geocoding_provider || PHOTON_URL - - const ret = await axios.get(`${geocoding_provider}`, { - params: { - q: details, - limit: 3, - }, - headers: { 'User-Agent': `gancio ${version}` } - }) - - // console.log(ret) - return res.json(ret.data) - - }, + } } diff --git a/server/api/index.js b/server/api/index.js index 32e15198..a2cecb4f 100644 --- a/server/api/index.js +++ b/server/api/index.js @@ -173,8 +173,8 @@ module.exports = () => { api.put('/place', isAdmin, placeController.updatePlace) // - GEOCODING - api.get('/placeOSM/Nominatim/:place_details', helpers.isGeocodingEnabled, geolocationController.apiLimit, placeController._nominatim) - api.get('/placeOSM/Photon/:place_details', helpers.isGeocodingEnabled, geolocationController.apiLimit, placeController._photon) + api.get('/placeOSM/Nominatim/:place_details', helpers.isGeocodingEnabled, geolocationController.instanceApiRateLimiter, geolocationController.providerRateLimit, geolocationController._nominatim) + api.get('/placeOSM/Photon/:place_details', helpers.isGeocodingEnabled, geolocationController.instanceApiRateLimiter, geolocationController.providerRateLimit, geolocationController._photon) // - TAGS api.get('/tags', isAdmin, tagController.getAll) diff --git a/yarn.lock b/yarn.lock index 2445ccf5..a5d03c8d 100644 --- a/yarn.lock +++ b/yarn.lock @@ -7950,6 +7950,11 @@ memoizee@^0.4.15: next-tick "^1.1.0" timers-ext "^0.1.7" +memory-cache@^0.2.0: + version "0.2.0" + resolved "https://registry.yarnpkg.com/memory-cache/-/memory-cache-0.2.0.tgz#7890b01d52c00c8ebc9d533e1f8eb17e3034871a" + integrity sha512-OcjA+jzjOYzKmKS6IQVALHLVz+rNTMPoJvCztFaZxwG14wtAW7VRZjwTQu06vKCYOxh4jVnik7ya0SXTB0W+xA== + memory-fs@^0.4.1: version "0.4.1" resolved "https://registry.yarnpkg.com/memory-fs/-/memory-fs-0.4.1.tgz#3a9a20b8462523e447cfbc7e8bb80ed667bfc552"