From 5b2f1986668737bd6167f64368ac4ce23e6e25b2 Mon Sep 17 00:00:00 2001 From: Marc Sladek Date: Wed, 1 Dec 2021 15:59:49 +0100 Subject: [PATCH 1/4] CelHibernateStoreCollectionPart remove stats --- .../part/CelHibernateStoreCollectionPart.java | 35 +++++-------------- 1 file changed, 9 insertions(+), 26 deletions(-) diff --git a/celements-model/src/main/java/com/celements/store/part/CelHibernateStoreCollectionPart.java b/celements-model/src/main/java/com/celements/store/part/CelHibernateStoreCollectionPart.java index 680ac633c..949202c6e 100644 --- a/celements-model/src/main/java/com/celements/store/part/CelHibernateStoreCollectionPart.java +++ b/celements-model/src/main/java/com/celements/store/part/CelHibernateStoreCollectionPart.java @@ -30,7 +30,6 @@ import com.xpn.xwiki.objects.classes.BaseClass; import com.xpn.xwiki.objects.classes.StringClass; import com.xpn.xwiki.objects.classes.TextAreaClass; -import com.xpn.xwiki.stats.impl.XWikiStats; //TODO CELDEV-626 - CelHibernateStore refactoring public class CelHibernateStoreCollectionPart { @@ -52,8 +51,6 @@ public void saveXWikiCollection(BaseCollection object, XWikiContext context, boo throw new XWikiException(MODULE_XWIKI_STORE, ERROR_XWIKI_STORE_HIBERNATE_SAVING_OBJECT, "Unable to save object with invalid id: " + object); } - // We need a slightly different behavior here - boolean stats = (object instanceof XWikiStats); boolean commit = false; try { if (bTransaction) { @@ -63,29 +60,14 @@ public void saveXWikiCollection(BaseCollection object, XWikiContext context, boo Session session = store.getSession(context); // Verify if the property already exists - Query query; - if (stats) { - query = session.createQuery("select obj.id from " + object.getClass().getName() - + " as obj where obj.id = :id"); - } else { - query = session.createQuery("select obj.id from BaseObject as obj where obj.id = :id"); - } + Query query = session.createQuery("select obj.id from BaseObject as obj where obj.id = :id"); query.setLong("id", object.getId()); if (query.uniqueResult() == null) { - if (stats) { - session.save(object); - } else { - session.save("com.xpn.xwiki.objects.BaseObject", object); - } + session.save("com.xpn.xwiki.objects.BaseObject", object); } else { - if (stats) { - session.update(object); - } else { - session.update("com.xpn.xwiki.objects.BaseObject", object); - } + session.update("com.xpn.xwiki.objects.BaseObject", object); } /* - * if (stats) session.saveOrUpdate(object); else * session.saveOrUpdate((String)"com.xpn.xwiki.objects.BaseObject", (Object)object); */ BaseClass bclass = object.getXClass(context); @@ -110,8 +92,8 @@ public void saveXWikiCollection(BaseCollection object, XWikiContext context, boo if (object.getXClassReference() != null) { // Remove all existing properties if (object.getFieldsToRemove().size() > 0) { - for (int i = 0; i < object.getFieldsToRemove().size(); i++) { - BaseProperty prop = (BaseProperty) object.getFieldsToRemove().get(i); + for (Object element : object.getFieldsToRemove()) { + BaseProperty prop = (BaseProperty) element; if (!handledProps.contains(prop.getName())) { session.delete(prop); } @@ -250,7 +232,8 @@ public void loadXWikiCollection(BaseCollection object1, XWikiDocument doc, XWiki } catch (Exception e2) { throw new XWikiException(MODULE_XWIKI_STORE, ERROR_XWIKI_STORE_HIBERNATE_LOADING_OBJECT, "Exception while loading property '" - + name + "' for object: " + object, e); + + name + "' for object: " + object, + e); } } @@ -297,8 +280,8 @@ public void deleteXWikiCollection(BaseCollection object, XWikiContext context, } if (object.getXClassReference() != null) { - for (Iterator it = object.getFieldList().iterator(); it.hasNext();) { - BaseElement property = (BaseElement) it.next(); + for (Object name : object.getFieldList()) { + BaseElement property = (BaseElement) name; if (!handledProps.contains(property.getName())) { if (evict) { session.evict(property); From 57f1a7cc9849b806809b1c51a1ffec9b53e65263 Mon Sep 17 00:00:00 2001 From: Marc Sladek Date: Wed, 1 Dec 2021 16:00:30 +0100 Subject: [PATCH 2/4] QueryExecutionService#executeReadSql with action --- .../query/IQueryExecutionServiceRole.java | 26 ++++++---- .../query/QueryExecutionService.java | 50 +++++++++++-------- 2 files changed, 45 insertions(+), 31 deletions(-) diff --git a/celements-model/src/main/java/com/celements/query/IQueryExecutionServiceRole.java b/celements-model/src/main/java/com/celements/query/IQueryExecutionServiceRole.java index d6baba14d..a9882ff50 100644 --- a/celements-model/src/main/java/com/celements/query/IQueryExecutionServiceRole.java +++ b/celements-model/src/main/java/com/celements/query/IQueryExecutionServiceRole.java @@ -2,7 +2,9 @@ import java.util.List; import java.util.Map; +import java.util.function.Consumer; +import javax.annotation.Nullable; import javax.validation.constraints.NotNull; import org.xwiki.component.annotation.ComponentRole; @@ -17,26 +19,32 @@ public interface IQueryExecutionServiceRole { @NotNull - public List> executeReadSql(@NotNull String sql) throws XWikiException; + List> executeReadSql(@NotNull String sql) throws XWikiException; @NotNull - public List> executeReadSql(@NotNull Class type, @NotNull String sql) + List> executeReadSql(@NotNull Class type, @NotNull String sql) throws XWikiException; - public int executeWriteSQL(String sql) throws XWikiException; + void executeReadSql(@NotNull Class type, @NotNull String sql, + @NotNull Consumer> action) throws XWikiException; - public List executeWriteSQLs(List sqls) throws XWikiException; + int executeWriteSQL(@NotNull String sql) throws XWikiException; - public int executeWriteHQL(String hql, Map binds) throws XWikiException; + List executeWriteSQLs(@NotNull List sqls) throws XWikiException; - public int executeWriteHQL(String hql, Map binds, WikiReference wikiRef) + int executeWriteHQL(@NotNull String hql, @NotNull Map binds) throws XWikiException; - public DocumentReference executeAndGetDocRef(Query query) throws QueryException; + int executeWriteHQL(@NotNull String hql, @NotNull Map binds, + @Nullable WikiReference wikiRef) throws XWikiException; - public List executeAndGetDocRefs(Query query) throws QueryException; + @Nullable + DocumentReference executeAndGetDocRef(@NotNull Query query) throws QueryException; - public boolean existsIndex(@NotNull WikiReference wikiRef, @NotNull String table, + @NotNull + List executeAndGetDocRefs(@NotNull Query query) throws QueryException; + + boolean existsIndex(@NotNull WikiReference wikiRef, @NotNull String table, @NotNull String name) throws XWikiException; } diff --git a/celements-model/src/main/java/com/celements/query/QueryExecutionService.java b/celements-model/src/main/java/com/celements/query/QueryExecutionService.java index abd8f125c..94a547f6f 100644 --- a/celements-model/src/main/java/com/celements/query/QueryExecutionService.java +++ b/celements-model/src/main/java/com/celements/query/QueryExecutionService.java @@ -3,11 +3,16 @@ import static com.google.common.base.MoreObjects.*; import static com.google.common.base.Preconditions.*; import static com.google.common.base.Strings.*; +import static java.util.stream.Collectors.*; import java.util.ArrayList; import java.util.Arrays; +import java.util.Iterator; import java.util.List; import java.util.Map; +import java.util.Objects; +import java.util.function.Consumer; +import java.util.stream.Stream; import org.hibernate.HibernateException; import org.hibernate.Session; @@ -23,6 +28,7 @@ import com.celements.model.access.ContextExecutor; import com.celements.model.context.ModelContext; +import com.celements.model.reference.RefBuilder; import com.celements.model.util.ModelUtils; import com.google.common.base.Strings; import com.xpn.xwiki.XWikiException; @@ -47,11 +53,21 @@ public List> executeReadSql(String sql) throws XWikiException { @Override public List> executeReadSql(Class type, String sql) throws XWikiException { + List> result = new ArrayList<>(); + executeReadSql(type, sql, result::add); + return result; + } + + @Override + public void executeReadSql(Class type, String sql, Consumer> action) + throws XWikiException { Session session = null; try { session = getNewHibSession(); - List result = session.createSQLQuery(sql).list(); - return harmoniseResult(type, result); + Iterator iter = session.createSQLQuery(sql).iterate(); + while (iter.hasNext()) { + action.accept(parseRow(type, iter)); + } } catch (HibernateException | ClassCastException exc) { throw new XWikiException(0, 0, "error while executing or parsing sql", exc); } finally { @@ -61,21 +77,16 @@ public List> executeReadSql(Class type, String sql) throws XWikiE } } - private List> harmoniseResult(Class type, List result) - throws ClassCastException { - List> ret = new ArrayList<>(); - for (Object elem : result) { - List resultRow = new ArrayList<>(); - if ((elem != null) && elem.getClass().isArray()) { // multiple columns selected - for (Object col : ((Object[]) elem)) { - resultRow.add(cast(type, col)); - } - } else { // one column selected - resultRow.add(cast(type, elem)); + private List parseRow(Class type, Object obj) throws ClassCastException { + List row = new ArrayList<>(); + if ((obj != null) && obj.getClass().isArray()) { // multiple columns selected + for (Object col : ((Object[]) obj)) { + row.add(cast(type, col)); } - ret.add(resultRow); + } else { // one column selected + row.add(cast(type, obj)); } - return ret; + return row; } private T cast(Class type, Object elem) throws ClassCastException { @@ -153,12 +164,7 @@ protected Integer call() throws XWikiException { @Override public DocumentReference executeAndGetDocRef(Query query) throws QueryException { - DocumentReference ret = null; - List list = executeAndGetDocRefs(query); - if (list.size() > 0) { - ret = list.get(0); - } - return ret; + return executeAndGetDocRefs(query).stream().findFirst().orElse(null); } @Override @@ -189,7 +195,7 @@ public boolean existsIndex(WikiReference wikiRef, String table, String name) throws XWikiException { String sql = getIndexExistSql(modelUtils.getDatabaseName(wikiRef), checkNotNull(emptyToNull( table)), checkNotNull(emptyToNull(name))); - return executeReadSql(String.class, sql).size() > 0; + return !executeReadSql(String.class, sql).isEmpty(); } private String getIndexExistSql(String database, String table, String name) { From 653238eb7197161cbe9e4c3f409141e6b39657c4 Mon Sep 17 00:00:00 2001 From: Marc Sladek Date: Wed, 1 Dec 2021 16:01:04 +0100 Subject: [PATCH 3/4] implement BaseObjectIdMigration --- .../migration/BaseObjectIdMigration.java | 224 ++++++++++++++++++ 1 file changed, 224 insertions(+) create mode 100644 celements-model/src/main/java/com/celements/model/migration/BaseObjectIdMigration.java diff --git a/celements-model/src/main/java/com/celements/model/migration/BaseObjectIdMigration.java b/celements-model/src/main/java/com/celements/model/migration/BaseObjectIdMigration.java new file mode 100644 index 000000000..79f8d2561 --- /dev/null +++ b/celements-model/src/main/java/com/celements/model/migration/BaseObjectIdMigration.java @@ -0,0 +1,224 @@ +package com.celements.model.migration; + +import java.util.List; +import java.util.concurrent.atomic.AtomicLong; +import java.util.concurrent.atomic.AtomicReference; + +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.xwiki.component.annotation.Component; +import org.xwiki.component.annotation.Requirement; +import org.xwiki.model.reference.DocumentReference; + +import com.celements.common.lambda.LambdaExceptionUtil.ThrowingSupplier; +import com.celements.migrations.SubSystemHibernateMigrationManager; +import com.celements.migrator.AbstractCelementsHibernateMigrator; +import com.celements.model.access.IModelAccessFacade; +import com.celements.model.access.exception.DocumentNotExistsException; +import com.celements.model.context.ModelContext; +import com.celements.model.util.ModelUtils; +import com.celements.query.IQueryExecutionServiceRole; +import com.celements.store.id.CelementsIdComputer; +import com.celements.store.id.CelementsIdComputer.IdComputationException; +import com.celements.store.id.IdVersion; +import com.google.common.primitives.Ints; +import com.google.common.primitives.Longs; +import com.xpn.xwiki.XWikiContext; +import com.xpn.xwiki.XWikiException; +import com.xpn.xwiki.doc.XWikiDocument; +import com.xpn.xwiki.doc.XWikiLock; +import com.xpn.xwiki.objects.BaseObject; +import com.xpn.xwiki.objects.classes.BaseClass; +import com.xpn.xwiki.store.XWikiHibernateStore; +import com.xpn.xwiki.store.hibernate.HibernateSessionFactory; +import com.xpn.xwiki.store.migration.XWikiDBVersion; + +@Component(BaseObjectIdMigration.NAME) +public class BaseObjectIdMigration extends AbstractCelementsHibernateMigrator { + + private static final Logger LOGGER = LoggerFactory.getLogger( + BaseObjectIdMigration.class); + + public static final String NAME = "BaseObjectIdMigration"; + + @Requirement + private HibernateSessionFactory sessionFactory; + + @Requirement + private IQueryExecutionServiceRole queryExecutor; + + @Requirement + private CelementsIdComputer idComputer; + + @Requirement + private IModelAccessFacade modelAccess; + + @Requirement + private ModelUtils modelUtils; + + @Requirement + private ModelContext context; + + @Override + public String getName() { + return NAME; + } + + @Override + public String getDescription() { + return "migrates ids of BaseObjects from " + IdVersion.XWIKI_2.name() + " to " + + idComputer.getIdVersion().name(); + } + + /** + * getVersion is using days since 1.1.2010 until the day of committing this migration + * 30.1.2021 -> 4351 http://www.wolframalpha.com/input/?i=days+since+01.01.2010 + */ + @Override + public XWikiDBVersion getVersion() { + return new XWikiDBVersion(4351); + } + + @Override + public void migrate(SubSystemHibernateMigrationManager manager, XWikiContext context) + throws XWikiException { + LOGGER.info("[{}] migrating object ids", context.getDatabase()); + try { + migrateAllObjectIds(context); + } catch (Exception exc) { + LOGGER.error("[{}] failed to migrate object ids", context.getDatabase(), exc); + throw exc; + } + } + + private void migrateAllObjectIds(XWikiContext context) throws XWikiException { + AtomicLong count = new AtomicLong(0); + AtomicReference previousDoc = new AtomicReference<>(); + try { + queryExecutor.executeReadSql(String.class, getSelectAllObjsWithOldIdSql(), row -> { + BaseObject object = createXObjectFromRow(row, context); + if (object != null) { + XWikiDocument doc = getAndLockNextDoc(object.getDocumentReference(), previousDoc.get()); + if (doc != null) { + previousDoc.set(doc); + boolean commited = inTransaction(() -> migrateObjectId(doc, object, context), context); + if (commited && ((count.incrementAndGet() % 10000) == 0)) { + LOGGER.info("[{}] migrated {}", context.getDatabase(), count.get()); + } + } + } + }); + } finally { + releaseLock(previousDoc.get()); // release the last doc + } + LOGGER.info("[{}] migrated {} object ids", context.getDatabase(), count); + } + + private XWikiDocument getAndLockNextDoc(DocumentReference docRef, XWikiDocument previousDoc) { + XWikiDocument nextDoc = null; + if ((previousDoc != null) && docRef.equals(previousDoc.getDocumentReference())) { + nextDoc = previousDoc; + } else { + try { + releaseLock(previousDoc); + } catch (XWikiException xwe) { + LOGGER.info("failed to release lock for [{}]", previousDoc.getDocumentReference()); + } + try { + nextDoc = acquireLock(modelAccess.getDocument(docRef)); + } catch (DocumentNotExistsException dne) { + LOGGER.warn("doc for object doesn't exist [{}]", docRef, dne); + } catch (XWikiException xwe) { + LOGGER.warn("failed acquiring lock for [{}]", docRef); + } + } + return nextDoc; + + } + + private boolean migrateObjectId(XWikiDocument doc, BaseObject object, XWikiContext context) + throws XWikiException { + try { + getHibStore().loadXWikiCollection(object, doc, context, false, false); + getHibStore().deleteXWikiObject(object, context, false); + long newId = idComputer.computeNextObjectId(doc); + if (object.getId() != newId) { + object.setId(newId, idComputer.getIdVersion()); + getHibStore().saveXWikiCollection(object, context, false); + return true; + } + return false; + } catch (IdComputationException exc) { + throw new XWikiException(0, 0, exc.getMessage(), exc); + } + } + + private boolean inTransaction(ThrowingSupplier action, + XWikiContext context) { + boolean commit = false; + try { + getHibStore().beginTransaction(context); + commit = action.get(); + } catch (XWikiException xwe) { + LOGGER.error("action failed", xwe); + } finally { + getHibStore().endTransaction(context, commit); + } + return commit; + } + + private XWikiDocument acquireLock(XWikiDocument doc) throws XWikiException { + XWikiLock lock = doc.getLock(context.getXWikiContext()); + if ((lock == null) || NAME.equals(lock.getUserName())) { + doc.setLock(NAME, context.getXWikiContext()); + return doc; + } + throw new XWikiException(0, 0, "lock already acquired by " + lock.getUserName()); + } + + private void releaseLock(XWikiDocument doc) throws XWikiException { + if (doc != null) { + XWikiLock lock = doc.getLock(context.getXWikiContext()); + if ((lock != null) && NAME.equals(lock.getUserName())) { + doc.removeLock(context.getXWikiContext()); + } else { + throw new XWikiException(0, 0, "lock not held by migration, instead held by: " + + ((lock != null) ? lock.getUserName() : "noone")); + } + } + } + + static String getSelectAllObjsWithOldIdSql() { + return "select XWO_ID, XWO_NAME, XWO_CLASSNAME, XWO_NUMBER from xwikiobjects" + + " where XWO_ID_VERSION = " + IdVersion.XWIKI_2.ordinal() + + " order by XWO_NAME, XWO_CLASSNAME, XWO_NUMBER"; + } + + /** + * + * @param row + * [XWO_ID, XWO_NAME, XWO_CLASSNAME, XWO_NUMBER] + */ + private BaseObject createXObjectFromRow(List row, XWikiContext context) { + try { + long id = Longs.tryParse(row.get(0)); + DocumentReference docRef = modelUtils.resolveRef(row.get(1), DocumentReference.class); + DocumentReference xClassRef = modelUtils.resolveRef(row.get(2), DocumentReference.class); + int nb = Ints.tryParse(row.get(3)); + BaseObject object = BaseClass.newCustomClassInstance(xClassRef, context); + object.setXClassReference(xClassRef); + object.setDocumentReference(docRef); + object.setNumber(nb); + object.setId(id, IdVersion.XWIKI_2); + return object; + } catch (IllegalArgumentException | XWikiException exc) { + LOGGER.warn("unable to parse illegal row [{}]", row, exc); + return null; + } + } + + private XWikiHibernateStore getHibStore() { + return context.getXWikiContext().getWiki().getHibernateStore(); + } + +} From 23cb179869b48e0a54ac05e15d8617499dbed784 Mon Sep 17 00:00:00 2001 From: Marc Sladek Date: Wed, 1 Dec 2021 16:45:33 +0100 Subject: [PATCH 4/4] improve migrateObjectId --- .../celements/model/migration/BaseObjectIdMigration.java | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/celements-model/src/main/java/com/celements/model/migration/BaseObjectIdMigration.java b/celements-model/src/main/java/com/celements/model/migration/BaseObjectIdMigration.java index 79f8d2561..048fb4db3 100644 --- a/celements-model/src/main/java/com/celements/model/migration/BaseObjectIdMigration.java +++ b/celements-model/src/main/java/com/celements/model/migration/BaseObjectIdMigration.java @@ -139,12 +139,13 @@ private XWikiDocument getAndLockNextDoc(DocumentReference docRef, XWikiDocument private boolean migrateObjectId(XWikiDocument doc, BaseObject object, XWikiContext context) throws XWikiException { try { - getHibStore().loadXWikiCollection(object, doc, context, false, false); - getHibStore().deleteXWikiObject(object, context, false); long newId = idComputer.computeNextObjectId(doc); - if (object.getId() != newId) { + if ((object.getId() != newId) || (idComputer.getIdVersion() != object.getIdVersion())) { + getHibStore().loadXWikiCollection(object, doc, context, false, false); + getHibStore().deleteXWikiObject(object, context, false); object.setId(newId, idComputer.getIdVersion()); getHibStore().saveXWikiCollection(object, context, false); + LOGGER.debug("migrated [{}]", object); return true; } return false;