Retrieving list of nodes & creating relationships

Drivers & Stacks Python
24

HMmmm no idea, maybe its the py2neo ? All I did was send 100k {id:"vdsvds"}dict with message to create relation to 1 node. No idea why he would not execute that command. Its the "simplest" thing I can think off to test after creating the 100k nodes in 1st place.

How can I debug this? To see why it did not make the relationships?

25

Someting I do when I work on a new Cypher request is that I first test it in Neo4j Desktop and when it works correctly, I adapt it to my Python program:)

26

Hmmm its strange, when I make 20 nodes relationships it work, 500, it works, 100k it does not

27

You tried 100k in Neo4j Desktop or in Python?
Do there is an error message?

28

In python, using the same code that generated the 100k nodes and was able to connect 20 nodes, but at 100k it breaks. No error code. just hangs for ever.

29

Could we see the code?

30

I've cleaned it up, will try running it again today, but for now >

import uuid
from json import dumps, loads

from py2neo import Node, Relationship, Graph, NodeMatcher
import time

graph = Graph("http://localhost:7474", password="graph")
globalStart = time.time()
print(graph)

def make3Manual():
    print("Deleting old graph")
    graph.delete_all()
    print("Deleted old graph")

    nData = {}
    nData["type"] = ""
    nData["name"] = "name"
    nData["reference"] = 1
    nData["icId"] = ""

    nodeSize = 100000 * [None]
    matSize = 100 * [None]
    containerSize = 100 * [None]
    sceneSize = 200 * [None]

    for id in range(0, len(nodeSize)):
        n = nData.copy()
        n["type"] = "Obj"
        n["name"] = "geo_" + str(id)
        n["refType"] = 1
        n["icId"] = str(uuid.uuid4())

        nodeSize[id] = n

    for id in range(0, len(matSize)):
        n = nData.copy()
        n["type"] = "animal"
        n["name"] = "ani_" + str(id)
        n["refType"] = 1
        n["icId"] = str(uuid.uuid4())

        matSize[id] = n

    for id in range(0, len(containerSize)):
        n = nData.copy()
        n["type"] = "Cont"
        n["name"] = "Cont_" + str(id)
        n["refType"] = 1
        n["icId"] = str(uuid.uuid4())

        containerSize[id] = n

    for id in range(0, len(sceneSize)):
        n = nData.copy()
        n["type"] = "Scene"
        n["name"] = "someani_" + str(id)
        n["refType"] = 1
        n["icId"] = str(uuid.uuid4())
        sceneSize[id] = n

    allData = nodeSize + matSize + containerSize + sceneSize
    print("Allocated arrays")
    tx = graph.begin()
    start = time.time()

    tx.run('''
    UNWIND $mapEntry AS mItem
    CALL apoc.create.node([mItem["type"]], {name:mItem["name"], art:mItem["atr"],icId:mItem["icId"], refType:mItem["refType"]})
    YIELD node
    RETURN node
                 '''
           , mapEntry=allData)

    tx.commit()
    print("Processed in : ", time.time() - start)


def getNodes(nodeClass, lim=-1):
    nMatcher = NodeMatcher(graph)
    if (lim):
        a = nMatcher.match(nodeClass).limit(lim)
    else:
        a = nMatcher.match(nodeClass)
    print("Found nodes : ", len(a), "  ", nodeClass)
    return list(a)


def buildRelation(source, key, dest):
    rela = len(source) * [None]
    x = 0
    for src in source:
        rela[x] = Relationship(src, key, dest)
        x = x + 1
    tx = graph.begin()
    for r in rela:
        tx.create(r)
    tx.commit()


def makeConnectionQuery(source, key, dest, sourceType, destType):
    for s in source:
        r = graph.run('''
        MATCH a={s}, b={d}
        CREATE UNIQUE (a)-[r-{k}]-()b
        RETURN a,b
        '''.format(s=s, k=key, d=dest))
        print("new con, key : ", key, " Reply : ", r)


def getNodesByType(type, limit=-1):
    if limit > 0:
        a = graph.run('''
        MATCH (n:{type}) 
        RETURN n 
        LIMIT {limit}'''.format(type=type, limit=limit))
        return a
    return graph.run('''
    MATCH (n:{type})
    RETURN n'''.format(type=type))


def makeConnection(source, key, dest, sourceType, destType):
    data = len(source) * [None]
    for a, item in enumerate(source):
        data[a] = {"id": item[0]["icId"]}
    print(dest[0]["icId"])
    print(data[0]["id"])
    print("processing count : ", len(source))
    tx = graph.begin()  #
    cmd = '''
        MATCH (dest:{destTag}{{icId:"{destId}"}})
        UNWIND $arrSource AS row
        MATCH (source:{srcType}{{icId:row.id}})
        
        CALL apoc.create.relationship(source, "{k}", {{relation:"usedIn"}}, dest)
        YIELD rel
        RETURN rel
                         '''.format(destId=dest[0]["icId"], destTag=destType, srcType=sourceType, k=key)
    #        WHERE NOT EXISTS((source)-[:usedIn]->(dest))

    print(cmd)
    c = tx.run(cmd, arrSource=data)
    print("con result : ", c)
    tx.commit()


print("Building DB")
make3Manual()

print("Getting data back")

geos = getNodesByType("Obj").to_ndarray()
containers = getNodesByType("Cont", 3).to_ndarray()
print(containers)
makeConnection(geos, "usedIn", containers[0], "Obj", "Cont")

print("End job!")
globalEnd = time.time()
print("Done in :", globalEnd - globalStart)
31

You sent 100k in the same batch? You sould try to iterate the batch 100 times with 1k nodes :)

32

Yeh, I figured it would be better to send it all at once ? Else why batch... :- )

33

You should definitely use a batch to create nodes and relations:)

34

But not big batch because it will not work ? :- )))

35

@Dariusz1989 Yeah, it can not work in some case when th database is receiving too much things to create, try to send several batches instead of one big:)