org.restheart.mongodb.services.CsvLoader Maven / Gradle / Ivy
Go to download
Show more of this group Show more artifacts with this name
Show all versions of restheart-mongodb Show documentation
Show all versions of restheart-mongodb Show documentation
RESTHeart MongoDB - MongoDB plugin
/*-
* ========================LICENSE_START=================================
* restheart-mongodb
* %%
* Copyright (C) 2014 - 2024 SoftInstigate
* %%
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see
* - db=<db_name> *required
* - coll=<collection_name> *required
* - id=<id_column_index> optional (default: no _id column, each row will
* get an new ObjectId)
* - sep=<column_separator> optional (default: ,)
* - props=<props> optional (default: no props) additional props to add to
* each row
* - values=<values> optional (default: no values) values of additional
* props to add to each row
* defined in conf file) of a tranformer to apply to imported data - update
* optional (default: no).use data to update matching documents");
*
* @author Andrea Di Cesare {@literal }
*/
@SuppressWarnings("unchecked")
@RegisterPlugin(name = "csvLoader", description = "Uploads a csv file in a MongoDb collection", secure = true, defaultURI = "/csv")
public class CsvLoader implements Service {
/**
*
*/
public static final String FILTER_PROPERTY = "_filter";
private static final String ERROR_QPARAM = "query parameters: " + "db= *required, "
+ "coll= *required, "
+ "id= optional (default: no _id column, each row will get an new ObjectId), "
+ "sep= optional (default: ,), "
+ "props= optional (default: no props) additional props to add to each row, "
+ "values= optional (default: no values) values of additional props to add to each row, "
+ "transformer= optional (default: no transformer). name of an interceptor to transform data, "
+ "update= optional (default: false). if true, update matching documents (requires id to be set), "
+ "upsert= optional (default: true). when update=true, create new documents when not matching existing ones.";
private static final String ERROR_NO_ID = "id must be set when update=true";
private static final String ERROR_WRONG_METHOD = "Only POST method is supported";
private final static FindOneAndUpdateOptions FAU_NO_UPSERT_OPS = new FindOneAndUpdateOptions().upsert(false);
private final static FindOneAndUpdateOptions FAU_WITH_UPSERT_OPS = new FindOneAndUpdateOptions().upsert(true);
@Inject("mclient")
MongoClient mclient;
/**
*
* @throws Exception
*/
@Override
public void handle(BsonFromCsvRequest request, BsonResponse response) throws Exception {
var exchange = request.getExchange();
if (request.isOptions()) {
handleOptions(request);
return;
}
if (!request.isPost()) {
response.setInError(HttpStatus.SC_NOT_IMPLEMENTED, ERROR_WRONG_METHOD);
return;
}
response.setContentTypeAsJson();
try {
var params = new CsvRequestParams(exchange);
if (params.db == null) {
response.setInError(HttpStatus.SC_BAD_REQUEST, "db qparam is mandatory");
return;
}
if (params.coll == null) {
response.setInError(HttpStatus.SC_BAD_REQUEST, "coll qparam is mandatory");
return;
}
if (params.update && params.idIdx < 0) {
response.setInError(HttpStatus.SC_BAD_REQUEST, ERROR_NO_ID);
} else {
var documents = request.getContent();
if (documents != null && documents.size() > 0) {
var mcoll = RHMongoClients.mclient().getDatabase(params.db).getCollection(params.coll, BsonDocument.class);
if (params.update && !params.upsert) {
documents.stream()
.map(doc -> doc.asDocument())
// add props specified via keys and values qparams
.map(doc -> addProps(params, doc))
.forEach(doc -> {
var updateQuery = new BsonDocument("_id", doc.remove("_id"));
// for upate import, take _filter property into account
// for instance, a filter allows to use $ positional array operator
var _filter = doc.remove(FILTER_PROPERTY);
if (_filter != null && _filter.isDocument()) {
updateQuery.putAll(_filter.asDocument());
}
if (params.upsert) {
mcoll.findOneAndUpdate(updateQuery, new BsonDocument("$set", doc), FAU_WITH_UPSERT_OPS);
} else {
mcoll.findOneAndUpdate(updateQuery, new BsonDocument("$set", doc), FAU_NO_UPSERT_OPS);
}
});
} else if (params.update && params.upsert) {
documents.stream()
.map(doc -> doc.asDocument())
// add props specified via keys and values qparams
.map(doc -> addProps(params, doc))
.forEach(doc -> {
var updateQuery = new BsonDocument("_id", doc.remove("_id"));
mcoll.findOneAndUpdate(updateQuery, new BsonDocument("$set", doc), FAU_WITH_UPSERT_OPS);
});
} else {
var docList = documents.stream()
.map(doc -> doc.asDocument())
// add props specified via keys and values qparams
.map(doc -> addProps(params, doc))
.collect(Collectors.toList());
mcoll.insertMany(docList);
}
response.setStatusCode(HttpStatus.SC_OK);
} else {
response.setStatusCode(HttpStatus.SC_NOT_MODIFIED);
}
}
} catch (IllegalArgumentException iae) {
response.setInError(HttpStatus.SC_BAD_REQUEST, ERROR_QPARAM);
}
}
private BsonDocument addProps(CsvRequestParams params, BsonDocument doc) {
if (params.props != null && params.values != null) {
@SuppressWarnings("rawtypes")
Deque _props = new ArrayDeque(params.props);
@SuppressWarnings("rawtypes")
Deque _values = new ArrayDeque(params.values);
while (!_props.isEmpty() && !_values.isEmpty()) {
doc.append(_props.pop(), getBsonValue(_values.poll()));
}
}
return doc;
}
private BsonValue getBsonValue(String raw) {
try {
return BsonUtils.parse(raw);
} catch (JsonParseException jpe) {
return new BsonString(raw);
}
}
@Override
public Consumer requestInitializer() {
return e -> BsonFromCsvRequest.init(e);
}
@Override
public Consumer responseInitializer() {
return e -> BsonResponse.init(e);
}
@Override
public Function request() {
return e -> BsonFromCsvRequest.of(e);
}
@Override
public Function response() {
return e -> BsonResponse.of(e);
}
}
class CsvRequestParams {
private static final String ID_IDX_QPARAM_NAME = "id";
private static final String SEPARATOR_QPARAM_NAME = "sep";
private static final String DB_QPARAM_NAME = "db";
private static final String COLL_QPARAM_NAME = "coll";
private static final String PROP_KEYS_NAME = "props";
private static final String PROP_VALUES_NAME = "values";
private static final String UPDATE_QPARAM_NAME = "update";
private static final String UPSERT_QPARAM_NAME = "upsert";
public final int idIdx;
public final String db;
public final String coll;
public final String sep;
public final boolean update;
public final boolean upsert;
public final Deque props;
public final Deque values;
CsvRequestParams(HttpServerExchange exchange) {
Deque _db = exchange.getQueryParameters().get(DB_QPARAM_NAME);
Deque _coll = exchange.getQueryParameters().get(COLL_QPARAM_NAME);
Deque _sep = exchange.getQueryParameters().get(SEPARATOR_QPARAM_NAME);
Deque _id = exchange.getQueryParameters().get(ID_IDX_QPARAM_NAME);
Deque _update = exchange.getQueryParameters().get(UPDATE_QPARAM_NAME);
Deque _upsert = exchange.getQueryParameters().get(UPSERT_QPARAM_NAME);
this.props = exchange.getQueryParameters().get(PROP_KEYS_NAME);
this.values = exchange.getQueryParameters().get(PROP_VALUES_NAME);
db = _db != null ? _db.size() > 0 ? _db.getFirst() : null : null;
coll = _coll != null ? _coll.size() > 0 ? _coll.getFirst() : null : null;
sep = _sep != null ? _sep.size() > 0 ? _sep.getFirst() : "" : ",";
String _idIdx = _id != null ? _id.size() > 0 ? _id.getFirst() : "-1" : "-1";
try {
idIdx = Integer.parseInt(_idIdx);
} catch (NumberFormatException nfe) {
throw new IllegalArgumentException(nfe);
}
update = _update != null && (_update.isEmpty() || "true".equalsIgnoreCase(_update.getFirst()));
upsert = _upsert == null || _update == null || _update.isEmpty() || "true".equalsIgnoreCase(_upsert.getFirst());
}
}
© 2015 - 2024 Weber Informatics LLC | Privacy Policy